Commit 16008d641670571ff4cd750b416c7caf2d89f467

Authored by Linus Torvalds

Merge branch 'for-3.3/drivers' of git://git.kernel.dk/linux-block

* 'for-3.3/drivers' of git://git.kernel.dk/linux-block:
  mtip32xx: do rebuild monitoring asynchronously
  xen-blkfront: Use kcalloc instead of kzalloc to allocate array
  mtip32xx: uninitialized variable in mtip_quiesce_io()
  mtip32xx: updates based on feedback
  xen-blkback: convert hole punching to discard request on loop devices
  xen/blkback: Move processing of BLKIF_OP_DISCARD from dispatch_rw_block_io
  xen/blk[front|back]: Enhance discard support with secure erasing support.
  xen/blk[front|back]: Squash blkif_request_rw and blkif_request_discard together
  mtip32xx: update to new ->make_request() API
  mtip32xx: add module.h include to avoid conflict with moduleh tree
  mtip32xx: mark a few more items static
  mtip32xx: ensure that all local functions are static
  mtip32xx: cleanup compat ioctl handling
  mtip32xx: fix warnings/errors on 32-bit compiles
  block: Add driver for Micron RealSSD pcie flash cards

Showing 11 changed files Side-by-side Diff

drivers/block/Kconfig
... ... @@ -116,6 +116,8 @@
116 116  
117 117 source "drivers/block/paride/Kconfig"
118 118  
  119 +source "drivers/block/mtip32xx/Kconfig"
  120 +
119 121 config BLK_CPQ_DA
120 122 tristate "Compaq SMART2 support"
121 123 depends on PCI && VIRT_TO_BUS
drivers/block/Makefile
... ... @@ -39,6 +39,7 @@
39 39 obj-$(CONFIG_XEN_BLKDEV_BACKEND) += xen-blkback/
40 40 obj-$(CONFIG_BLK_DEV_DRBD) += drbd/
41 41 obj-$(CONFIG_BLK_DEV_RBD) += rbd.o
  42 +obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx/
42 43  
43 44 swim_mod-y := swim.o swim_asm.o
drivers/block/mtip32xx/Kconfig
  1 +#
  2 +# mtip32xx device driver configuration
  3 +#
  4 +
  5 +config BLK_DEV_PCIESSD_MTIP32XX
  6 + tristate "Block Device Driver for Micron PCIe SSDs"
  7 + depends on HOTPLUG_PCI_PCIE
  8 + help
  9 + This enables the block driver for Micron PCIe SSDs.
drivers/block/mtip32xx/Makefile
  1 +#
  2 +# Makefile for Block device driver for Micron PCIe SSD
  3 +#
  4 +
  5 +obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx.o
drivers/block/mtip32xx/mtip32xx.c
Changes suppressed. Click to show
  1 +/*
  2 + * Driver for the Micron P320 SSD
  3 + * Copyright (C) 2011 Micron Technology, Inc.
  4 + *
  5 + * Portions of this code were derived from works subjected to the
  6 + * following copyright:
  7 + * Copyright (C) 2009 Integrated Device Technology, Inc.
  8 + *
  9 + * This program is free software; you can redistribute it and/or modify
  10 + * it under the terms of the GNU General Public License as published by
  11 + * the Free Software Foundation; either version 2 of the License, or
  12 + * (at your option) any later version.
  13 + *
  14 + * This program is distributed in the hope that it will be useful,
  15 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17 + * GNU General Public License for more details.
  18 + *
  19 + */
  20 +
  21 +#include <linux/pci.h>
  22 +#include <linux/interrupt.h>
  23 +#include <linux/ata.h>
  24 +#include <linux/delay.h>
  25 +#include <linux/hdreg.h>
  26 +#include <linux/uaccess.h>
  27 +#include <linux/random.h>
  28 +#include <linux/smp.h>
  29 +#include <linux/compat.h>
  30 +#include <linux/fs.h>
  31 +#include <linux/module.h>
  32 +#include <linux/genhd.h>
  33 +#include <linux/blkdev.h>
  34 +#include <linux/bio.h>
  35 +#include <linux/dma-mapping.h>
  36 +#include <linux/idr.h>
  37 +#include <linux/kthread.h>
  38 +#include <../drivers/ata/ahci.h>
  39 +#include "mtip32xx.h"
  40 +
  41 +#define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32)
  42 +#define HW_CMD_TBL_SZ (AHCI_CMD_TBL_HDR_SZ + (MTIP_MAX_SG * 16))
  43 +#define HW_CMD_TBL_AR_SZ (HW_CMD_TBL_SZ * MTIP_MAX_COMMAND_SLOTS)
  44 +#define HW_PORT_PRIV_DMA_SZ \
  45 + (HW_CMD_SLOT_SZ + HW_CMD_TBL_AR_SZ + AHCI_RX_FIS_SZ)
  46 +
  47 +#define HOST_HSORG 0xFC
  48 +#define HSORG_DISABLE_SLOTGRP_INTR (1<<24)
  49 +#define HSORG_DISABLE_SLOTGRP_PXIS (1<<16)
  50 +#define HSORG_HWREV 0xFF00
  51 +#define HSORG_STYLE 0x8
  52 +#define HSORG_SLOTGROUPS 0x7
  53 +
  54 +#define PORT_COMMAND_ISSUE 0x38
  55 +#define PORT_SDBV 0x7C
  56 +
  57 +#define PORT_OFFSET 0x100
  58 +#define PORT_MEM_SIZE 0x80
  59 +
  60 +#define PORT_IRQ_ERR \
  61 + (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \
  62 + PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \
  63 + PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \
  64 + PORT_IRQ_OVERFLOW)
  65 +#define PORT_IRQ_LEGACY \
  66 + (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS)
  67 +#define PORT_IRQ_HANDLED \
  68 + (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \
  69 + PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \
  70 + PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)
  71 +#define DEF_PORT_IRQ \
  72 + (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS)
  73 +
  74 +/* product numbers */
  75 +#define MTIP_PRODUCT_UNKNOWN 0x00
  76 +#define MTIP_PRODUCT_ASICFPGA 0x11
  77 +
  78 +/* Device instance number, incremented each time a device is probed. */
  79 +static int instance;
  80 +
  81 +/*
  82 + * Global variable used to hold the major block device number
  83 + * allocated in mtip_init().
  84 + */
  85 +static int mtip_major;
  86 +
  87 +static DEFINE_SPINLOCK(rssd_index_lock);
  88 +static DEFINE_IDA(rssd_index_ida);
  89 +
  90 +static int mtip_block_initialize(struct driver_data *dd);
  91 +
  92 +#ifdef CONFIG_COMPAT
  93 +struct mtip_compat_ide_task_request_s {
  94 + __u8 io_ports[8];
  95 + __u8 hob_ports[8];
  96 + ide_reg_valid_t out_flags;
  97 + ide_reg_valid_t in_flags;
  98 + int data_phase;
  99 + int req_cmd;
  100 + compat_ulong_t out_size;
  101 + compat_ulong_t in_size;
  102 +};
  103 +#endif
  104 +
  105 +/*
  106 + * This function check_for_surprise_removal is called
  107 + * while card is removed from the system and it will
  108 + * read the vendor id from the configration space
  109 + *
  110 + * @pdev Pointer to the pci_dev structure.
  111 + *
  112 + * return value
  113 + * true if device removed, else false
  114 + */
  115 +static bool mtip_check_surprise_removal(struct pci_dev *pdev)
  116 +{
  117 + u16 vendor_id = 0;
  118 +
  119 + /* Read the vendorID from the configuration space */
  120 + pci_read_config_word(pdev, 0x00, &vendor_id);
  121 + if (vendor_id == 0xFFFF)
  122 + return true; /* device removed */
  123 +
  124 + return false; /* device present */
  125 +}
  126 +
  127 +/*
  128 + * This function is called for clean the pending command in the
  129 + * command slot during the surprise removal of device and return
  130 + * error to the upper layer.
  131 + *
  132 + * @dd Pointer to the DRIVER_DATA structure.
  133 + *
  134 + * return value
  135 + * None
  136 + */
  137 +static void mtip_command_cleanup(struct driver_data *dd)
  138 +{
  139 + int group = 0, commandslot = 0, commandindex = 0;
  140 + struct mtip_cmd *command;
  141 + struct mtip_port *port = dd->port;
  142 +
  143 + for (group = 0; group < 4; group++) {
  144 + for (commandslot = 0; commandslot < 32; commandslot++) {
  145 + if (!(port->allocated[group] & (1 << commandslot)))
  146 + continue;
  147 +
  148 + commandindex = group << 5 | commandslot;
  149 + command = &port->commands[commandindex];
  150 +
  151 + if (atomic_read(&command->active)
  152 + && (command->async_callback)) {
  153 + command->async_callback(command->async_data,
  154 + -ENODEV);
  155 + command->async_callback = NULL;
  156 + command->async_data = NULL;
  157 + }
  158 +
  159 + dma_unmap_sg(&port->dd->pdev->dev,
  160 + command->sg,
  161 + command->scatter_ents,
  162 + command->direction);
  163 + }
  164 + }
  165 +
  166 + up(&port->cmd_slot);
  167 +
  168 + atomic_set(&dd->drv_cleanup_done, true);
  169 +}
  170 +
  171 +/*
  172 + * Obtain an empty command slot.
  173 + *
  174 + * This function needs to be reentrant since it could be called
  175 + * at the same time on multiple CPUs. The allocation of the
  176 + * command slot must be atomic.
  177 + *
  178 + * @port Pointer to the port data structure.
  179 + *
  180 + * return value
  181 + * >= 0 Index of command slot obtained.
  182 + * -1 No command slots available.
  183 + */
  184 +static int get_slot(struct mtip_port *port)
  185 +{
  186 + int slot, i;
  187 + unsigned int num_command_slots = port->dd->slot_groups * 32;
  188 +
  189 + /*
  190 + * Try 10 times, because there is a small race here.
  191 + * that's ok, because it's still cheaper than a lock.
  192 + *
  193 + * Race: Since this section is not protected by lock, same bit
  194 + * could be chosen by different process contexts running in
  195 + * different processor. So instead of costly lock, we are going
  196 + * with loop.
  197 + */
  198 + for (i = 0; i < 10; i++) {
  199 + slot = find_next_zero_bit(port->allocated,
  200 + num_command_slots, 1);
  201 + if ((slot < num_command_slots) &&
  202 + (!test_and_set_bit(slot, port->allocated)))
  203 + return slot;
  204 + }
  205 + dev_warn(&port->dd->pdev->dev, "Failed to get a tag.\n");
  206 +
  207 + if (mtip_check_surprise_removal(port->dd->pdev)) {
  208 + /* Device not present, clean outstanding commands */
  209 + mtip_command_cleanup(port->dd);
  210 + }
  211 + return -1;
  212 +}
  213 +
  214 +/*
  215 + * Release a command slot.
  216 + *
  217 + * @port Pointer to the port data structure.
  218 + * @tag Tag of command to release
  219 + *
  220 + * return value
  221 + * None
  222 + */
  223 +static inline void release_slot(struct mtip_port *port, int tag)
  224 +{
  225 + smp_mb__before_clear_bit();
  226 + clear_bit(tag, port->allocated);
  227 + smp_mb__after_clear_bit();
  228 +}
  229 +
  230 +/*
  231 + * Reset the HBA (without sleeping)
  232 + *
  233 + * Just like hba_reset, except does not call sleep, so can be
  234 + * run from interrupt/tasklet context.
  235 + *
  236 + * @dd Pointer to the driver data structure.
  237 + *
  238 + * return value
  239 + * 0 The reset was successful.
  240 + * -1 The HBA Reset bit did not clear.
  241 + */
  242 +static int hba_reset_nosleep(struct driver_data *dd)
  243 +{
  244 + unsigned long timeout;
  245 +
  246 + /* Chip quirk: quiesce any chip function */
  247 + mdelay(10);
  248 +
  249 + /* Set the reset bit */
  250 + writel(HOST_RESET, dd->mmio + HOST_CTL);
  251 +
  252 + /* Flush */
  253 + readl(dd->mmio + HOST_CTL);
  254 +
  255 + /*
  256 + * Wait 10ms then spin for up to 1 second
  257 + * waiting for reset acknowledgement
  258 + */
  259 + timeout = jiffies + msecs_to_jiffies(1000);
  260 + mdelay(10);
  261 + while ((readl(dd->mmio + HOST_CTL) & HOST_RESET)
  262 + && time_before(jiffies, timeout))
  263 + mdelay(1);
  264 +
  265 + if (readl(dd->mmio + HOST_CTL) & HOST_RESET)
  266 + return -1;
  267 +
  268 + return 0;
  269 +}
  270 +
  271 +/*
  272 + * Issue a command to the hardware.
  273 + *
  274 + * Set the appropriate bit in the s_active and Command Issue hardware
  275 + * registers, causing hardware command processing to begin.
  276 + *
  277 + * @port Pointer to the port structure.
  278 + * @tag The tag of the command to be issued.
  279 + *
  280 + * return value
  281 + * None
  282 + */
  283 +static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag)
  284 +{
  285 + unsigned long flags = 0;
  286 +
  287 + atomic_set(&port->commands[tag].active, 1);
  288 +
  289 + spin_lock_irqsave(&port->cmd_issue_lock, flags);
  290 +
  291 + writel((1 << MTIP_TAG_BIT(tag)),
  292 + port->s_active[MTIP_TAG_INDEX(tag)]);
  293 + writel((1 << MTIP_TAG_BIT(tag)),
  294 + port->cmd_issue[MTIP_TAG_INDEX(tag)]);
  295 +
  296 + spin_unlock_irqrestore(&port->cmd_issue_lock, flags);
  297 +}
  298 +
  299 +/*
  300 + * Enable/disable the reception of FIS
  301 + *
  302 + * @port Pointer to the port data structure
  303 + * @enable 1 to enable, 0 to disable
  304 + *
  305 + * return value
  306 + * Previous state: 1 enabled, 0 disabled
  307 + */
  308 +static int mtip_enable_fis(struct mtip_port *port, int enable)
  309 +{
  310 + u32 tmp;
  311 +
  312 + /* enable FIS reception */
  313 + tmp = readl(port->mmio + PORT_CMD);
  314 + if (enable)
  315 + writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
  316 + else
  317 + writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
  318 +
  319 + /* Flush */
  320 + readl(port->mmio + PORT_CMD);
  321 +
  322 + return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX));
  323 +}
  324 +
  325 +/*
  326 + * Enable/disable the DMA engine
  327 + *
  328 + * @port Pointer to the port data structure
  329 + * @enable 1 to enable, 0 to disable
  330 + *
  331 + * return value
  332 + * Previous state: 1 enabled, 0 disabled.
  333 + */
  334 +static int mtip_enable_engine(struct mtip_port *port, int enable)
  335 +{
  336 + u32 tmp;
  337 +
  338 + /* enable FIS reception */
  339 + tmp = readl(port->mmio + PORT_CMD);
  340 + if (enable)
  341 + writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD);
  342 + else
  343 + writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD);
  344 +
  345 + readl(port->mmio + PORT_CMD);
  346 + return (((tmp & PORT_CMD_START) == PORT_CMD_START));
  347 +}
  348 +
  349 +/*
  350 + * Enables the port DMA engine and FIS reception.
  351 + *
  352 + * return value
  353 + * None
  354 + */
  355 +static inline void mtip_start_port(struct mtip_port *port)
  356 +{
  357 + /* Enable FIS reception */
  358 + mtip_enable_fis(port, 1);
  359 +
  360 + /* Enable the DMA engine */
  361 + mtip_enable_engine(port, 1);
  362 +}
  363 +
  364 +/*
  365 + * Deinitialize a port by disabling port interrupts, the DMA engine,
  366 + * and FIS reception.
  367 + *
  368 + * @port Pointer to the port structure
  369 + *
  370 + * return value
  371 + * None
  372 + */
  373 +static inline void mtip_deinit_port(struct mtip_port *port)
  374 +{
  375 + /* Disable interrupts on this port */
  376 + writel(0, port->mmio + PORT_IRQ_MASK);
  377 +
  378 + /* Disable the DMA engine */
  379 + mtip_enable_engine(port, 0);
  380 +
  381 + /* Disable FIS reception */
  382 + mtip_enable_fis(port, 0);
  383 +}
  384 +
  385 +/*
  386 + * Initialize a port.
  387 + *
  388 + * This function deinitializes the port by calling mtip_deinit_port() and
  389 + * then initializes it by setting the command header and RX FIS addresses,
  390 + * clearing the SError register and any pending port interrupts before
  391 + * re-enabling the default set of port interrupts.
  392 + *
  393 + * @port Pointer to the port structure.
  394 + *
  395 + * return value
  396 + * None
  397 + */
  398 +static void mtip_init_port(struct mtip_port *port)
  399 +{
  400 + int i;
  401 + mtip_deinit_port(port);
  402 +
  403 + /* Program the command list base and FIS base addresses */
  404 + if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) {
  405 + writel((port->command_list_dma >> 16) >> 16,
  406 + port->mmio + PORT_LST_ADDR_HI);
  407 + writel((port->rxfis_dma >> 16) >> 16,
  408 + port->mmio + PORT_FIS_ADDR_HI);
  409 + }
  410 +
  411 + writel(port->command_list_dma & 0xFFFFFFFF,
  412 + port->mmio + PORT_LST_ADDR);
  413 + writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR);
  414 +
  415 + /* Clear SError */
  416 + writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
  417 +
  418 + /* reset the completed registers.*/
  419 + for (i = 0; i < port->dd->slot_groups; i++)
  420 + writel(0xFFFFFFFF, port->completed[i]);
  421 +
  422 + /* Clear any pending interrupts for this port */
  423 + writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT);
  424 +
  425 + /* Enable port interrupts */
  426 + writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK);
  427 +}
  428 +
  429 +/*
  430 + * Restart a port
  431 + *
  432 + * @port Pointer to the port data structure.
  433 + *
  434 + * return value
  435 + * None
  436 + */
  437 +static void mtip_restart_port(struct mtip_port *port)
  438 +{
  439 + unsigned long timeout;
  440 +
  441 + /* Disable the DMA engine */
  442 + mtip_enable_engine(port, 0);
  443 +
  444 + /* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */
  445 + timeout = jiffies + msecs_to_jiffies(500);
  446 + while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON)
  447 + && time_before(jiffies, timeout))
  448 + ;
  449 +
  450 + /*
  451 + * Chip quirk: escalate to hba reset if
  452 + * PxCMD.CR not clear after 500 ms
  453 + */
  454 + if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) {
  455 + dev_warn(&port->dd->pdev->dev,
  456 + "PxCMD.CR not clear, escalating reset\n");
  457 +
  458 + if (hba_reset_nosleep(port->dd))
  459 + dev_err(&port->dd->pdev->dev,
  460 + "HBA reset escalation failed.\n");
  461 +
  462 + /* 30 ms delay before com reset to quiesce chip */
  463 + mdelay(30);
  464 + }
  465 +
  466 + dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n");
  467 +
  468 + /* Set PxSCTL.DET */
  469 + writel(readl(port->mmio + PORT_SCR_CTL) |
  470 + 1, port->mmio + PORT_SCR_CTL);
  471 + readl(port->mmio + PORT_SCR_CTL);
  472 +
  473 + /* Wait 1 ms to quiesce chip function */
  474 + timeout = jiffies + msecs_to_jiffies(1);
  475 + while (time_before(jiffies, timeout))
  476 + ;
  477 +
  478 + /* Clear PxSCTL.DET */
  479 + writel(readl(port->mmio + PORT_SCR_CTL) & ~1,
  480 + port->mmio + PORT_SCR_CTL);
  481 + readl(port->mmio + PORT_SCR_CTL);
  482 +
  483 + /* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */
  484 + timeout = jiffies + msecs_to_jiffies(500);
  485 + while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
  486 + && time_before(jiffies, timeout))
  487 + ;
  488 +
  489 + if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
  490 + dev_warn(&port->dd->pdev->dev,
  491 + "COM reset failed\n");
  492 +
  493 + /* Clear SError, the PxSERR.DIAG.x should be set so clear it */
  494 + writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
  495 +
  496 + /* Enable the DMA engine */
  497 + mtip_enable_engine(port, 1);
  498 +}
  499 +
  500 +/*
  501 + * Called periodically to see if any read/write commands are
  502 + * taking too long to complete.
  503 + *
  504 + * @data Pointer to the PORT data structure.
  505 + *
  506 + * return value
  507 + * None
  508 + */
  509 +static void mtip_timeout_function(unsigned long int data)
  510 +{
  511 + struct mtip_port *port = (struct mtip_port *) data;
  512 + struct host_to_dev_fis *fis;
  513 + struct mtip_cmd *command;
  514 + int tag, cmdto_cnt = 0;
  515 + unsigned int bit, group;
  516 + unsigned int num_command_slots = port->dd->slot_groups * 32;
  517 +
  518 + if (unlikely(!port))
  519 + return;
  520 +
  521 + if (atomic_read(&port->dd->resumeflag) == true) {
  522 + mod_timer(&port->cmd_timer,
  523 + jiffies + msecs_to_jiffies(30000));
  524 + return;
  525 + }
  526 +
  527 + for (tag = 0; tag < num_command_slots; tag++) {
  528 + /*
  529 + * Skip internal command slot as it has
  530 + * its own timeout mechanism
  531 + */
  532 + if (tag == MTIP_TAG_INTERNAL)
  533 + continue;
  534 +
  535 + if (atomic_read(&port->commands[tag].active) &&
  536 + (time_after(jiffies, port->commands[tag].comp_time))) {
  537 + group = tag >> 5;
  538 + bit = tag & 0x1F;
  539 +
  540 + command = &port->commands[tag];
  541 + fis = (struct host_to_dev_fis *) command->command;
  542 +
  543 + dev_warn(&port->dd->pdev->dev,
  544 + "Timeout for command tag %d\n", tag);
  545 +
  546 + cmdto_cnt++;
  547 + if (cmdto_cnt == 1)
  548 + set_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags);
  549 +
  550 + /*
  551 + * Clear the completed bit. This should prevent
  552 + * any interrupt handlers from trying to retire
  553 + * the command.
  554 + */
  555 + writel(1 << bit, port->completed[group]);
  556 +
  557 + /* Call the async completion callback. */
  558 + if (likely(command->async_callback))
  559 + command->async_callback(command->async_data,
  560 + -EIO);
  561 + command->async_callback = NULL;
  562 + command->comp_func = NULL;
  563 +
  564 + /* Unmap the DMA scatter list entries */
  565 + dma_unmap_sg(&port->dd->pdev->dev,
  566 + command->sg,
  567 + command->scatter_ents,
  568 + command->direction);
  569 +
  570 + /*
  571 + * Clear the allocated bit and active tag for the
  572 + * command.
  573 + */
  574 + atomic_set(&port->commands[tag].active, 0);
  575 + release_slot(port, tag);
  576 +
  577 + up(&port->cmd_slot);
  578 + }
  579 + }
  580 +
  581 + if (cmdto_cnt) {
  582 + dev_warn(&port->dd->pdev->dev,
  583 + "%d commands timed out: restarting port",
  584 + cmdto_cnt);
  585 + mtip_restart_port(port);
  586 + clear_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags);
  587 + wake_up_interruptible(&port->svc_wait);
  588 + }
  589 +
  590 + /* Restart the timer */
  591 + mod_timer(&port->cmd_timer,
  592 + jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD));
  593 +}
  594 +
  595 +/*
  596 + * IO completion function.
  597 + *
  598 + * This completion function is called by the driver ISR when a
  599 + * command that was issued by the kernel completes. It first calls the
  600 + * asynchronous completion function which normally calls back into the block
  601 + * layer passing the asynchronous callback data, then unmaps the
  602 + * scatter list associated with the completed command, and finally
  603 + * clears the allocated bit associated with the completed command.
  604 + *
  605 + * @port Pointer to the port data structure.
  606 + * @tag Tag of the command.
  607 + * @data Pointer to driver_data.
  608 + * @status Completion status.
  609 + *
  610 + * return value
  611 + * None
  612 + */
  613 +static void mtip_async_complete(struct mtip_port *port,
  614 + int tag,
  615 + void *data,
  616 + int status)
  617 +{
  618 + struct mtip_cmd *command;
  619 + struct driver_data *dd = data;
  620 + int cb_status = status ? -EIO : 0;
  621 +
  622 + if (unlikely(!dd) || unlikely(!port))
  623 + return;
  624 +
  625 + command = &port->commands[tag];
  626 +
  627 + if (unlikely(status == PORT_IRQ_TF_ERR)) {
  628 + dev_warn(&port->dd->pdev->dev,
  629 + "Command tag %d failed due to TFE\n", tag);
  630 + }
  631 +
  632 + /* Upper layer callback */
  633 + if (likely(command->async_callback))
  634 + command->async_callback(command->async_data, cb_status);
  635 +
  636 + command->async_callback = NULL;
  637 + command->comp_func = NULL;
  638 +
  639 + /* Unmap the DMA scatter list entries */
  640 + dma_unmap_sg(&dd->pdev->dev,
  641 + command->sg,
  642 + command->scatter_ents,
  643 + command->direction);
  644 +
  645 + /* Clear the allocated and active bits for the command */
  646 + atomic_set(&port->commands[tag].active, 0);
  647 + release_slot(port, tag);
  648 +
  649 + up(&port->cmd_slot);
  650 +}
  651 +
  652 +/*
  653 + * Internal command completion callback function.
  654 + *
  655 + * This function is normally called by the driver ISR when an internal
  656 + * command completed. This function signals the command completion by
  657 + * calling complete().
  658 + *
  659 + * @port Pointer to the port data structure.
  660 + * @tag Tag of the command that has completed.
  661 + * @data Pointer to a completion structure.
  662 + * @status Completion status.
  663 + *
  664 + * return value
  665 + * None
  666 + */
  667 +static void mtip_completion(struct mtip_port *port,
  668 + int tag,
  669 + void *data,
  670 + int status)
  671 +{
  672 + struct mtip_cmd *command = &port->commands[tag];
  673 + struct completion *waiting = data;
  674 + if (unlikely(status == PORT_IRQ_TF_ERR))
  675 + dev_warn(&port->dd->pdev->dev,
  676 + "Internal command %d completed with TFE\n", tag);
  677 +
  678 + command->async_callback = NULL;
  679 + command->comp_func = NULL;
  680 +
  681 + complete(waiting);
  682 +}
  683 +
  684 +/*
  685 + * Helper function for tag logging
  686 + */
  687 +static void print_tags(struct driver_data *dd,
  688 + char *msg,
  689 + unsigned long *tagbits)
  690 +{
  691 + unsigned int tag, count = 0;
  692 +
  693 + for (tag = 0; tag < (dd->slot_groups) * 32; tag++) {
  694 + if (test_bit(tag, tagbits))
  695 + count++;
  696 + }
  697 + if (count)
  698 + dev_info(&dd->pdev->dev, "%s [%i tags]\n", msg, count);
  699 +}
  700 +
  701 +/*
  702 + * Handle an error.
  703 + *
  704 + * @dd Pointer to the DRIVER_DATA structure.
  705 + *
  706 + * return value
  707 + * None
  708 + */
  709 +static void mtip_handle_tfe(struct driver_data *dd)
  710 +{
  711 + int group, tag, bit, reissue;
  712 + struct mtip_port *port;
  713 + struct mtip_cmd *command;
  714 + u32 completed;
  715 + struct host_to_dev_fis *fis;
  716 + unsigned long tagaccum[SLOTBITS_IN_LONGS];
  717 +
  718 + dev_warn(&dd->pdev->dev, "Taskfile error\n");
  719 +
  720 + port = dd->port;
  721 +
  722 + /* Stop the timer to prevent command timeouts. */
  723 + del_timer(&port->cmd_timer);
  724 +
  725 + /* Set eh_active */
  726 + set_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags);
  727 +
  728 + /* Loop through all the groups */
  729 + for (group = 0; group < dd->slot_groups; group++) {
  730 + completed = readl(port->completed[group]);
  731 +
  732 + /* clear completed status register in the hardware.*/
  733 + writel(completed, port->completed[group]);
  734 +
  735 + /* clear the tag accumulator */
  736 + memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
  737 +
  738 + /* Process successfully completed commands */
  739 + for (bit = 0; bit < 32 && completed; bit++) {
  740 + if (!(completed & (1<<bit)))
  741 + continue;
  742 + tag = (group << 5) + bit;
  743 +
  744 + /* Skip the internal command slot */
  745 + if (tag == MTIP_TAG_INTERNAL)
  746 + continue;
  747 +
  748 + command = &port->commands[tag];
  749 + if (likely(command->comp_func)) {
  750 + set_bit(tag, tagaccum);
  751 + atomic_set(&port->commands[tag].active, 0);
  752 + command->comp_func(port,
  753 + tag,
  754 + command->comp_data,
  755 + 0);
  756 + } else {
  757 + dev_err(&port->dd->pdev->dev,
  758 + "Missing completion func for tag %d",
  759 + tag);
  760 + if (mtip_check_surprise_removal(dd->pdev)) {
  761 + mtip_command_cleanup(dd);
  762 + /* don't proceed further */
  763 + return;
  764 + }
  765 + }
  766 + }
  767 + }
  768 + print_tags(dd, "TFE tags completed:", tagaccum);
  769 +
  770 + /* Restart the port */
  771 + mdelay(20);
  772 + mtip_restart_port(port);
  773 +
  774 + /* clear the tag accumulator */
  775 + memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
  776 +
  777 + /* Loop through all the groups */
  778 + for (group = 0; group < dd->slot_groups; group++) {
  779 + for (bit = 0; bit < 32; bit++) {
  780 + reissue = 1;
  781 + tag = (group << 5) + bit;
  782 +
  783 + /* If the active bit is set re-issue the command */
  784 + if (atomic_read(&port->commands[tag].active) == 0)
  785 + continue;
  786 +
  787 + fis = (struct host_to_dev_fis *)
  788 + port->commands[tag].command;
  789 +
  790 + /* Should re-issue? */
  791 + if (tag == MTIP_TAG_INTERNAL ||
  792 + fis->command == ATA_CMD_SET_FEATURES)
  793 + reissue = 0;
  794 +
  795 + /*
  796 + * First check if this command has
  797 + * exceeded its retries.
  798 + */
  799 + if (reissue &&
  800 + (port->commands[tag].retries-- > 0)) {
  801 +
  802 + set_bit(tag, tagaccum);
  803 +
  804 + /* Update the timeout value. */
  805 + port->commands[tag].comp_time =
  806 + jiffies + msecs_to_jiffies(
  807 + MTIP_NCQ_COMMAND_TIMEOUT_MS);
  808 + /* Re-issue the command. */
  809 + mtip_issue_ncq_command(port, tag);
  810 +
  811 + continue;
  812 + }
  813 +
  814 + /* Retire a command that will not be reissued */
  815 + dev_warn(&port->dd->pdev->dev,
  816 + "retiring tag %d\n", tag);
  817 + atomic_set(&port->commands[tag].active, 0);
  818 +
  819 + if (port->commands[tag].comp_func)
  820 + port->commands[tag].comp_func(
  821 + port,
  822 + tag,
  823 + port->commands[tag].comp_data,
  824 + PORT_IRQ_TF_ERR);
  825 + else
  826 + dev_warn(&port->dd->pdev->dev,
  827 + "Bad completion for tag %d\n",
  828 + tag);
  829 + }
  830 + }
  831 + print_tags(dd, "TFE tags reissued:", tagaccum);
  832 +
  833 + /* clear eh_active */
  834 + clear_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags);
  835 + wake_up_interruptible(&port->svc_wait);
  836 +
  837 + mod_timer(&port->cmd_timer,
  838 + jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD));
  839 +}
  840 +
  841 +/*
  842 + * Handle a set device bits interrupt
  843 + */
  844 +static inline void mtip_process_sdbf(struct driver_data *dd)
  845 +{
  846 + struct mtip_port *port = dd->port;
  847 + int group, tag, bit;
  848 + u32 completed;
  849 + struct mtip_cmd *command;
  850 +
  851 + /* walk all bits in all slot groups */
  852 + for (group = 0; group < dd->slot_groups; group++) {
  853 + completed = readl(port->completed[group]);
  854 +
  855 + /* clear completed status register in the hardware.*/
  856 + writel(completed, port->completed[group]);
  857 +
  858 + /* Process completed commands. */
  859 + for (bit = 0;
  860 + (bit < 32) && completed;
  861 + bit++, completed >>= 1) {
  862 + if (completed & 0x01) {
  863 + tag = (group << 5) | bit;
  864 +
  865 + /* skip internal command slot. */
  866 + if (unlikely(tag == MTIP_TAG_INTERNAL))
  867 + continue;
  868 +
  869 + command = &port->commands[tag];
  870 + /* make internal callback */
  871 + if (likely(command->comp_func)) {
  872 + command->comp_func(
  873 + port,
  874 + tag,
  875 + command->comp_data,
  876 + 0);
  877 + } else {
  878 + dev_warn(&dd->pdev->dev,
  879 + "Null completion "
  880 + "for tag %d",
  881 + tag);
  882 +
  883 + if (mtip_check_surprise_removal(
  884 + dd->pdev)) {
  885 + mtip_command_cleanup(dd);
  886 + return;
  887 + }
  888 + }
  889 + }
  890 + }
  891 + }
  892 +}
  893 +
  894 +/*
  895 + * Process legacy pio and d2h interrupts
  896 + */
  897 +static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat)
  898 +{
  899 + struct mtip_port *port = dd->port;
  900 + struct mtip_cmd *cmd = &port->commands[MTIP_TAG_INTERNAL];
  901 +
  902 + if (test_bit(MTIP_FLAG_IC_ACTIVE_BIT, &port->flags) &&
  903 + (cmd != NULL) && !(readl(port->cmd_issue[MTIP_TAG_INTERNAL])
  904 + & (1 << MTIP_TAG_INTERNAL))) {
  905 + if (cmd->comp_func) {
  906 + cmd->comp_func(port,
  907 + MTIP_TAG_INTERNAL,
  908 + cmd->comp_data,
  909 + 0);
  910 + return;
  911 + }
  912 + }
  913 +
  914 + dev_warn(&dd->pdev->dev, "IRQ status 0x%x ignored.\n", port_stat);
  915 +
  916 + return;
  917 +}
  918 +
  919 +/*
  920 + * Demux and handle errors
  921 + */
  922 +static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat)
  923 +{
  924 + if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR)))
  925 + mtip_handle_tfe(dd);
  926 +
  927 + if (unlikely(port_stat & PORT_IRQ_CONNECT)) {
  928 + dev_warn(&dd->pdev->dev,
  929 + "Clearing PxSERR.DIAG.x\n");
  930 + writel((1 << 26), dd->port->mmio + PORT_SCR_ERR);
  931 + }
  932 +
  933 + if (unlikely(port_stat & PORT_IRQ_PHYRDY)) {
  934 + dev_warn(&dd->pdev->dev,
  935 + "Clearing PxSERR.DIAG.n\n");
  936 + writel((1 << 16), dd->port->mmio + PORT_SCR_ERR);
  937 + }
  938 +
  939 + if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) {
  940 + dev_warn(&dd->pdev->dev,
  941 + "Port stat errors %x unhandled\n",
  942 + (port_stat & ~PORT_IRQ_HANDLED));
  943 + }
  944 +}
  945 +
  946 +static inline irqreturn_t mtip_handle_irq(struct driver_data *data)
  947 +{
  948 + struct driver_data *dd = (struct driver_data *) data;
  949 + struct mtip_port *port = dd->port;
  950 + u32 hba_stat, port_stat;
  951 + int rv = IRQ_NONE;
  952 +
  953 + hba_stat = readl(dd->mmio + HOST_IRQ_STAT);
  954 + if (hba_stat) {
  955 + rv = IRQ_HANDLED;
  956 +
  957 + /* Acknowledge the interrupt status on the port.*/
  958 + port_stat = readl(port->mmio + PORT_IRQ_STAT);
  959 + writel(port_stat, port->mmio + PORT_IRQ_STAT);
  960 +
  961 + /* Demux port status */
  962 + if (likely(port_stat & PORT_IRQ_SDB_FIS))
  963 + mtip_process_sdbf(dd);
  964 +
  965 + if (unlikely(port_stat & PORT_IRQ_ERR)) {
  966 + if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
  967 + mtip_command_cleanup(dd);
  968 + /* don't proceed further */
  969 + return IRQ_HANDLED;
  970 + }
  971 +
  972 + mtip_process_errors(dd, port_stat & PORT_IRQ_ERR);
  973 + }
  974 +
  975 + if (unlikely(port_stat & PORT_IRQ_LEGACY))
  976 + mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY);
  977 + }
  978 +
  979 + /* acknowledge interrupt */
  980 + writel(hba_stat, dd->mmio + HOST_IRQ_STAT);
  981 +
  982 + return rv;
  983 +}
  984 +
  985 +/*
  986 + * Wrapper for mtip_handle_irq
  987 + * (ignores return code)
  988 + */
  989 +static void mtip_tasklet(unsigned long data)
  990 +{
  991 + mtip_handle_irq((struct driver_data *) data);
  992 +}
  993 +
  994 +/*
  995 + * HBA interrupt subroutine.
  996 + *
  997 + * @irq IRQ number.
  998 + * @instance Pointer to the driver data structure.
  999 + *
  1000 + * return value
  1001 + * IRQ_HANDLED A HBA interrupt was pending and handled.
  1002 + * IRQ_NONE This interrupt was not for the HBA.
  1003 + */
  1004 +static irqreturn_t mtip_irq_handler(int irq, void *instance)
  1005 +{
  1006 + struct driver_data *dd = instance;
  1007 + tasklet_schedule(&dd->tasklet);
  1008 + return IRQ_HANDLED;
  1009 +}
  1010 +
  1011 +static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag)
  1012 +{
  1013 + atomic_set(&port->commands[tag].active, 1);
  1014 + writel(1 << MTIP_TAG_BIT(tag),
  1015 + port->cmd_issue[MTIP_TAG_INDEX(tag)]);
  1016 +}
  1017 +
  1018 +/*
  1019 + * Wait for port to quiesce
  1020 + *
  1021 + * @port Pointer to port data structure
  1022 + * @timeout Max duration to wait (ms)
  1023 + *
  1024 + * return value
  1025 + * 0 Success
  1026 + * -EBUSY Commands still active
  1027 + */
  1028 +static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout)
  1029 +{
  1030 + unsigned long to;
  1031 + unsigned int n;
  1032 + unsigned int active = 1;
  1033 +
  1034 + to = jiffies + msecs_to_jiffies(timeout);
  1035 + do {
  1036 + if (test_bit(MTIP_FLAG_SVC_THD_ACTIVE_BIT, &port->flags) &&