Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 16008d641670571ff4cd750b416c7caf2d89f467

Authored by Linus Torvalds 2012-01-16 04:48:41 +0800

Exists in master and in 6 other branches

Merge branch 'for-3.3/drivers' of git://git.kernel.dk/linux-block

* 'for-3.3/drivers' of git://git.kernel.dk/linux-block:
  mtip32xx: do rebuild monitoring asynchronously
  xen-blkfront: Use kcalloc instead of kzalloc to allocate array
  mtip32xx: uninitialized variable in mtip_quiesce_io()
  mtip32xx: updates based on feedback
  xen-blkback: convert hole punching to discard request on loop devices
  xen/blkback: Move processing of BLKIF_OP_DISCARD from dispatch_rw_block_io
  xen/blk[front|back]: Enhance discard support with secure erasing support.
  xen/blk[front|back]: Squash blkif_request_rw and blkif_request_discard together
  mtip32xx: update to new ->make_request() API
  mtip32xx: add module.h include to avoid conflict with moduleh tree
  mtip32xx: mark a few more items static
  mtip32xx: ensure that all local functions are static
  mtip32xx: cleanup compat ioctl handling
  mtip32xx: fix warnings/errors on 32-bit compiles
  block: Add driver for Micron RealSSD pcie flash cards

Showing 11 changed files Inline Diff

drivers/block/Kconfig
drivers/block/Makefile
drivers/block/mtip32xx/Kconfig
drivers/block/mtip32xx/Makefile
drivers/block/mtip32xx/mtip32xx.c
drivers/block/mtip32xx/mtip32xx.h
drivers/block/xen-blkback/blkback.c
drivers/block/xen-blkback/common.h
drivers/block/xen-blkback/xenbus.c
drivers/block/xen-blkfront.c
include/xen/interface/io/blkif.h

drivers/block/Kconfig

Diff comments View file @ 16008d6

 #
 # Block device driver configuration
 #
 menuconfig BLK_DEV
 	bool "Block devices"
 	depends on BLOCK
 	default y
 	---help---
 	  Say Y here to get to see options for various different block device
 	  drivers. This option alone does not add any kernel code.
 	  If you say N, all options in this submenu will be skipped and disabled;
 	  only do this if you know what you are doing.
 if BLK_DEV
 config BLK_DEV_FD
 	tristate "Normal floppy disk support"
 	depends on ARCH_MAY_HAVE_PC_FDC
 	---help---
 	  If you want to use the floppy disk drive(s) of your PC under Linux,
 	  say Y. Information about this driver, especially important for IBM
 	  Thinkpad users, is contained in
 	  <file:Documentation/blockdev/floppy.txt>.
 	  That file also contains the location of the Floppy driver FAQ as
 	  well as location of the fdutils package used to configure additional
 	  parameters of the driver at run time.
 	  To compile this driver as a module, choose M here: the
 	  module will be called floppy.
 config AMIGA_FLOPPY
 	tristate "Amiga floppy support"
 	depends on AMIGA
 config ATARI_FLOPPY
 	tristate "Atari floppy support"
 	depends on ATARI
 config MAC_FLOPPY
 	tristate "Support for PowerMac floppy"
 	depends on PPC_PMAC && !PPC_PMAC64
 	help
 	  If you have a SWIM-3 (Super Woz Integrated Machine 3; from Apple)
 	  floppy controller, say Y here. Most commonly found in PowerMacs.
 config BLK_DEV_SWIM
 	tristate "Support for SWIM Macintosh floppy"
 	depends on M68K && MAC
 	help
 	  You should select this option if you want floppy support
 	  and you don't have a II, IIfx, Q900, Q950 or AV series.
 config AMIGA_Z2RAM
 	tristate "Amiga Zorro II ramdisk support"
 	depends on ZORRO
 	help
 	  This enables support for using Chip RAM and Zorro II RAM as a
 	  ramdisk or as a swap partition. Say Y if you want to include this
 	  driver in the kernel.
 	  To compile this driver as a module, choose M here: the
 	  module will be called z2ram.
 config BLK_DEV_XD
 	tristate "XT hard disk support"
 	depends on ISA && ISA_DMA_API
 	select CHECK_SIGNATURE
 	help
 	  Very old 8 bit hard disk controllers used in the IBM XT computer
 	  will be supported if you say Y here.
 	  To compile this driver as a module, choose M here: the
 	  module will be called xd.
 	  It's pretty unlikely that you have one of these: say N.
 config GDROM
 	tristate "SEGA Dreamcast GD-ROM drive"
 	depends on SH_DREAMCAST
 	help
 	  A standard SEGA Dreamcast comes with a modified CD ROM drive called a
 	  "GD-ROM" by SEGA to signify it is capable of reading special disks
 	  with up to 1 GB of data. This drive will also read standard CD ROM
 	  disks. Select this option to access any disks in your GD ROM drive.
 	  Most users will want to say "Y" here.
 	  You can also build this as a module which will be called gdrom.
 config PARIDE
 	tristate "Parallel port IDE device support"
 	depends on PARPORT_PC
 	---help---
 	  There are many external CD-ROM and disk devices that connect through
 	  your computer's parallel port. Most of them are actually IDE devices
 	  using a parallel port IDE adapter. This option enables the PARIDE
 	  subsystem which contains drivers for many of these external drives.
 	  Read <file:Documentation/blockdev/paride.txt> for more information.
 	  If you have said Y to the "Parallel-port support" configuration
 	  option, you may share a single port between your printer and other
 	  parallel port devices. Answer Y to build PARIDE support into your
 	  kernel, or M if you would like to build it as a loadable module. If
 	  your parallel port support is in a loadable module, you must build
 	  PARIDE as a module. If you built PARIDE support into your kernel,
 	  you may still build the individual protocol modules and high-level
 	  drivers as loadable modules. If you build this support as a module,
 	  it will be called paride.
 	  To use the PARIDE support, you must say Y or M here and also to at
 	  least one high-level driver (e.g. "Parallel port IDE disks",
 	  "Parallel port ATAPI CD-ROMs", "Parallel port ATAPI disks" etc.) and
 	  to at least one protocol driver (e.g. "ATEN EH-100 protocol",
 	  "MicroSolutions backpack protocol", "DataStor Commuter protocol"
 	  etc.).
 source "drivers/block/paride/Kconfig"
+source "drivers/block/mtip32xx/Kconfig"
 config BLK_CPQ_DA
 	tristate "Compaq SMART2 support"
 	depends on PCI && VIRT_TO_BUS
 	help
 	  This is the driver for Compaq Smart Array controllers.  Everyone
 	  using these boards should say Y here.  See the file
 	  <file:Documentation/blockdev/cpqarray.txt> for the current list of
 	  boards supported by this driver, and for further information on the
 	  use of this driver.
 config BLK_CPQ_CISS_DA
 	tristate "Compaq Smart Array 5xxx support"
 	depends on PCI
 	help
 	  This is the driver for Compaq Smart Array 5xxx controllers.
 	  Everyone using these boards should say Y here.
 	  See <file:Documentation/blockdev/cciss.txt> for the current list of
 	  boards supported by this driver, and for further information
 	  on the use of this driver.
 config CISS_SCSI_TAPE
 	bool "SCSI tape drive support for Smart Array 5xxx"
 	depends on BLK_CPQ_CISS_DA && PROC_FS
 	depends on SCSI=y || SCSI=BLK_CPQ_CISS_DA
 	help
 	  When enabled (Y), this option allows SCSI tape drives and SCSI medium
 	  changers (tape robots) to be accessed via a Compaq 5xxx array
 	  controller.  (See <file:Documentation/blockdev/cciss.txt> for more details.)
 	  "SCSI support" and "SCSI tape support" must also be enabled for this
 	  option to work.
 	  When this option is disabled (N), the SCSI portion of the driver
 	  is not compiled.
 config BLK_DEV_DAC960
 	tristate "Mylex DAC960/DAC1100 PCI RAID Controller support"
 	depends on PCI
 	help
 	  This driver adds support for the Mylex DAC960, AcceleRAID, and
 	  eXtremeRAID PCI RAID controllers.  See the file
 	  <file:Documentation/blockdev/README.DAC960> for further information
 	  about this driver.
 	  To compile this driver as a module, choose M here: the
 	  module will be called DAC960.
 config BLK_DEV_UMEM
 	tristate "Micro Memory MM5415 Battery Backed RAM support (EXPERIMENTAL)"
 	depends on PCI && EXPERIMENTAL
 	---help---
 	  Saying Y here will include support for the MM5415 family of
 	  battery backed (Non-volatile) RAM cards.
 	  <http://www.umem.com/>
 	  The cards appear as block devices that can be partitioned into
 	  as many as 15 partitions.
 	  To compile this driver as a module, choose M here: the
 	  module will be called umem.
 	  The umem driver has not yet been allocated a MAJOR number, so
 	  one is chosen dynamically.
 config BLK_DEV_UBD
 	bool "Virtual block device"
 	depends on UML
 	---help---
           The User-Mode Linux port includes a driver called UBD which will let
           you access arbitrary files on the host computer as block devices.
           Unless you know that you do not need such virtual block devices say
           Y here.
 config BLK_DEV_UBD_SYNC
 	bool "Always do synchronous disk IO for UBD"
 	depends on BLK_DEV_UBD
 	---help---
 	  Writes to the virtual block device are not immediately written to the
 	  host's disk; this may cause problems if, for example, the User-Mode
 	  Linux 'Virtual Machine' uses a journalling filesystem and the host
 	  computer crashes.
           Synchronous operation (i.e. always writing data to the host's disk
           immediately) is configurable on a per-UBD basis by using a special
           kernel command line option.  Alternatively, you can say Y here to
           turn on synchronous operation by default for all block devices.
           If you're running a journalling file system (like reiserfs, for
           example) in your virtual machine, you will want to say Y here.  If
           you care for the safety of the data in your virtual machine, Y is a
           wise choice too.  In all other cases (for example, if you're just
           playing around with User-Mode Linux) you can choose N.
 config BLK_DEV_COW_COMMON
 	bool
 	default BLK_DEV_UBD
 config BLK_DEV_LOOP
 	tristate "Loopback device support"
 	---help---
 	  Saying Y here will allow you to use a regular file as a block
 	  device; you can then create a file system on that block device and
 	  mount it just as you would mount other block devices such as hard
 	  drive partitions, CD-ROM drives or floppy drives. The loop devices
 	  are block special device files with major number 7 and typically
 	  called /dev/loop0, /dev/loop1 etc.
 	  This is useful if you want to check an ISO 9660 file system before
 	  burning the CD, or if you want to use floppy images without first
 	  writing them to floppy. Furthermore, some Linux distributions avoid
 	  the need for a dedicated Linux partition by keeping their complete
 	  root file system inside a DOS FAT file using this loop device
 	  driver.
 	  To use the loop device, you need the losetup utility, found in the
 	  util-linux package, see
 	  <ftp://ftp.kernel.org/pub/linux/utils/util-linux/>.
 	  The loop device driver can also be used to "hide" a file system in
 	  a disk partition, floppy, or regular file, either using encryption
 	  (scrambling the data) or steganography (hiding the data in the low
 	  bits of, say, a sound file). This is also safe if the file resides
 	  on a remote file server.
 	  There are several ways of encrypting disks. Some of these require
 	  kernel patches. The vanilla kernel offers the cryptoloop option
 	  and a Device Mapper target (which is superior, as it supports all
 	  file systems). If you want to use the cryptoloop, say Y to both
 	  LOOP and CRYPTOLOOP, and make sure you have a recent (version 2.12
 	  or later) version of util-linux. Additionally, be aware that
 	  the cryptoloop is not safe for storing journaled filesystems.
 	  Note that this loop device has nothing to do with the loopback
 	  device used for network connections from the machine to itself.
 	  To compile this driver as a module, choose M here: the
 	  module will be called loop.
 	  Most users will answer N here.
 config BLK_DEV_LOOP_MIN_COUNT
 	int "Number of loop devices to pre-create at init time"
 	depends on BLK_DEV_LOOP
 	default 8
 	help
 	  Static number of loop devices to be unconditionally pre-created
 	  at init time.
 	  This default value can be overwritten on the kernel command
 	  line or with module-parameter loop.max_loop.
 	  The historic default is 8. If a late 2011 version of losetup(8)
 	  is used, it can be set to 0, since needed loop devices can be
 	  dynamically allocated with the /dev/loop-control interface.
 config BLK_DEV_CRYPTOLOOP
 	tristate "Cryptoloop Support"
 	select CRYPTO
 	select CRYPTO_CBC
 	depends on BLK_DEV_LOOP
 	---help---
 	  Say Y here if you want to be able to use the ciphers that are
 	  provided by the CryptoAPI as loop transformation. This might be
 	  used as hard disk encryption.
 	  WARNING: This device is not safe for journaled file systems like
 	  ext3 or Reiserfs. Please use the Device Mapper crypto module
 	  instead, which can be configured to be on-disk compatible with the
 	  cryptoloop device.
 source "drivers/block/drbd/Kconfig"
 config BLK_DEV_NBD
 	tristate "Network block device support"
 	depends on NET
 	---help---
 	  Saying Y here will allow your computer to be a client for network
 	  block devices, i.e. it will be able to use block devices exported by
 	  servers (mount file systems on them etc.). Communication between
 	  client and server works over TCP/IP networking, but to the client
 	  program this is hidden: it looks like a regular local file access to
 	  a block device special file such as /dev/nd0.
 	  Network block devices also allows you to run a block-device in
 	  userland (making server and client physically the same computer,
 	  communicating using the loopback network device).
 	  Read <file:Documentation/blockdev/nbd.txt> for more information,
 	  especially about where to find the server code, which runs in user
 	  space and does not need special kernel support.
 	  Note that this has nothing to do with the network file systems NFS
 	  or Coda; you can say N here even if you intend to use NFS or Coda.
 	  To compile this driver as a module, choose M here: the
 	  module will be called nbd.
 	  If unsure, say N.
 config BLK_DEV_OSD
 	tristate "OSD object-as-blkdev support"
 	depends on SCSI_OSD_ULD
 	---help---
 	  Saying Y or M here will allow the exporting of a single SCSI
 	  OSD (object-based storage) object as a Linux block device.
 	  For example, if you create a 2G object on an OSD device,
 	  you can then use this module to present that 2G object as
 	  a Linux block device.
 	  To compile this driver as a module, choose M here: the
 	  module will be called osdblk.
 	  If unsure, say N.
 config BLK_DEV_SX8
 	tristate "Promise SATA SX8 support"
 	depends on PCI
 	---help---
 	  Saying Y or M here will enable support for the
 	  Promise SATA SX8 controllers.
 	  Use devices /dev/sx8/$N and /dev/sx8/$Np$M.
 config BLK_DEV_UB
 	tristate "Low Performance USB Block driver"
 	depends on USB
 	help
 	  This driver supports certain USB attached storage devices
 	  such as flash keys.
 	  If you enable this driver, it is recommended to avoid conflicts
 	  with usb-storage by enabling USB_LIBUSUAL.
 	  If unsure, say N.
 config BLK_DEV_RAM
 	tristate "RAM block device support"
 	---help---
 	  Saying Y here will allow you to use a portion of your RAM memory as
 	  a block device, so that you can make file systems on it, read and
 	  write to it and do all the other things that you can do with normal
 	  block devices (such as hard drives). It is usually used to load and
 	  store a copy of a minimal root file system off of a floppy into RAM
 	  during the initial install of Linux.
 	  Note that the kernel command line option "ramdisk=XX" is now obsolete.
 	  For details, read <file:Documentation/blockdev/ramdisk.txt>.
 	  To compile this driver as a module, choose M here: the
 	  module will be called rd.
 	  Most normal users won't need the RAM disk functionality, and can
 	  thus say N here.
 config BLK_DEV_RAM_COUNT
 	int "Default number of RAM disks"
 	default "16"
 	depends on BLK_DEV_RAM
 	help
 	  The default value is 16 RAM disks. Change this if you know what you
 	  are doing. If you boot from a filesystem that needs to be extracted
 	  in memory, you will need at least one RAM disk (e.g. root on cramfs).
 config BLK_DEV_RAM_SIZE
 	int "Default RAM disk size (kbytes)"
 	depends on BLK_DEV_RAM
 	default "4096"
 	help
 	  The default value is 4096 kilobytes. Only change this if you know
 	  what you are doing.
 config BLK_DEV_XIP
 	bool "Support XIP filesystems on RAM block device"
 	depends on BLK_DEV_RAM
 	default n
 	help
 	  Support XIP filesystems (such as ext2 with XIP support on) on
 	  top of block ram device. This will slightly enlarge the kernel, and
 	  will prevent RAM block device backing store memory from being
 	  allocated from highmem (only a problem for highmem systems).
 config CDROM_PKTCDVD
 	tristate "Packet writing on CD/DVD media"
 	depends on !UML
 	help
 	  If you have a CDROM/DVD drive that supports packet writing, say
 	  Y to include support. It should work with any MMC/Mt Fuji
 	  compliant ATAPI or SCSI drive, which is just about any newer
 	  DVD/CD writer.
 	  Currently only writing to CD-RW, DVD-RW, DVD+RW and DVDRAM discs
 	  is possible.
 	  DVD-RW disks must be in restricted overwrite mode.
 	  See the file <file:Documentation/cdrom/packet-writing.txt>
 	  for further information on the use of this driver.
 	  To compile this driver as a module, choose M here: the
 	  module will be called pktcdvd.
 config CDROM_PKTCDVD_BUFFERS
 	int "Free buffers for data gathering"
 	depends on CDROM_PKTCDVD
 	default "8"
 	help
 	  This controls the maximum number of active concurrent packets. More
 	  concurrent packets can increase write performance, but also require
 	  more memory. Each concurrent packet will require approximately 64Kb
 	  of non-swappable kernel memory, memory which will be allocated when
 	  a disc is opened for writing.
 config CDROM_PKTCDVD_WCACHE
 	bool "Enable write caching (EXPERIMENTAL)"
 	depends on CDROM_PKTCDVD && EXPERIMENTAL
 	help
 	  If enabled, write caching will be set for the CD-R/W device. For now
 	  this option is dangerous unless the CD-RW media is known good, as we
 	  don't do deferred write error handling yet.
 config ATA_OVER_ETH
 	tristate "ATA over Ethernet support"
 	depends on NET
 	help
 	This driver provides Support for ATA over Ethernet block
 	devices like the Coraid EtherDrive (R) Storage Blade.
 config MG_DISK
 	tristate "mGine mflash, gflash support"
 	depends on ARM && GPIOLIB
 	help
 	  mGine mFlash(gFlash) block device driver
 config MG_DISK_RES
 	int "Size of reserved area before MBR"
 	depends on MG_DISK
 	default 0
 	help
 	  Define size of reserved area that usually used for boot. Unit is KB.
 	  All of the block device operation will be taken this value as start
 	  offset
 	  Examples:
 			1024 => 1 MB
 config SUNVDC
 	tristate "Sun Virtual Disk Client support"
 	depends on SUN_LDOMS
 	help
 	  Support for virtual disk devices as a client under Sun
 	  Logical Domains.
 source "drivers/s390/block/Kconfig"
 config XILINX_SYSACE
 	tristate "Xilinx SystemACE support"
 	depends on 4xx || MICROBLAZE
 	help
 	  Include support for the Xilinx SystemACE CompactFlash interface
 config XEN_BLKDEV_FRONTEND
 	tristate "Xen virtual block device support"
 	depends on XEN
 	default y
 	select XEN_XENBUS_FRONTEND
 	help
 	  This driver implements the front-end of the Xen virtual
 	  block device driver.  It communicates with a back-end driver
 	  in another domain which drives the actual block device.
 config XEN_BLKDEV_BACKEND
 	tristate "Xen block-device backend driver"
 	depends on XEN_BACKEND
 	help
 	  The block-device backend driver allows the kernel to export its
 	  block devices to other guests via a high-performance shared-memory
 	  interface.
 	  The corresponding Linux frontend driver is enabled by the
 	  CONFIG_XEN_BLKDEV_FRONTEND configuration option.
 	  The backend driver attaches itself to a any block device specified
 	  in the XenBus configuration. There are no limits to what the block
 	  device as long as it has a major and minor.
 	  If you are compiling a kernel to run in a Xen block backend driver
 	  domain (often this is domain 0) you should say Y here. To
 	  compile this driver as a module, chose M here: the module
 	  will be called xen-blkback.
 config VIRTIO_BLK
 	tristate "Virtio block driver (EXPERIMENTAL)"
 	depends on EXPERIMENTAL && VIRTIO
 	---help---
 	  This is the virtual block driver for virtio.  It can be used with
           lguest or QEMU based VMMs (like KVM or Xen).  Say Y or M.
 config BLK_DEV_HD
 	bool "Very old hard disk (MFM/RLL/IDE) driver"
 	depends on HAVE_IDE
 	depends on !ARM || ARCH_RPC || ARCH_SHARK || BROKEN
 	help
 	  This is a very old hard disk driver that lacks the enhanced
 	  functionality of the newer ones.
 	  It is required for systems with ancient MFM/RLL/ESDI drives.
 	  If unsure, say N.
 config BLK_DEV_RBD
 	tristate "Rados block device (RBD)"
 	depends on INET && EXPERIMENTAL && BLOCK
 	select CEPH_LIB
 	select LIBCRC32C
 	select CRYPTO_AES
 	select CRYPTO
 	default n
 	help
 	  Say Y here if you want include the Rados block device, which stripes
 	  a block device over objects stored in the Ceph distributed object
 	  store.
 	  More information at http://ceph.newdream.net/.
 	  If unsure, say N.
 endif # BLK_DEV

drivers/block/Makefile

Diff comments View file @ 16008d6

 #
 # Makefile for the kernel block device drivers.
 #
 # 12 June 2000, Christoph Hellwig <hch@infradead.org>
 # Rewritten to use lists instead of if-statements.
 #
 obj-$(CONFIG_MAC_FLOPPY)	+= swim3.o
 obj-$(CONFIG_BLK_DEV_SWIM)	+= swim_mod.o
 obj-$(CONFIG_BLK_DEV_FD)	+= floppy.o
 obj-$(CONFIG_AMIGA_FLOPPY)	+= amiflop.o
 obj-$(CONFIG_PS3_DISK)		+= ps3disk.o
 obj-$(CONFIG_PS3_VRAM)		+= ps3vram.o
 obj-$(CONFIG_ATARI_FLOPPY)	+= ataflop.o
 obj-$(CONFIG_AMIGA_Z2RAM)	+= z2ram.o
 obj-$(CONFIG_BLK_DEV_RAM)	+= brd.o
 obj-$(CONFIG_BLK_DEV_LOOP)	+= loop.o
 obj-$(CONFIG_BLK_DEV_XD)	+= xd.o
 obj-$(CONFIG_BLK_CPQ_DA)	+= cpqarray.o
 obj-$(CONFIG_BLK_CPQ_CISS_DA)  += cciss.o
 obj-$(CONFIG_BLK_DEV_DAC960)	+= DAC960.o
 obj-$(CONFIG_XILINX_SYSACE)	+= xsysace.o
 obj-$(CONFIG_CDROM_PKTCDVD)	+= pktcdvd.o
 obj-$(CONFIG_MG_DISK)		+= mg_disk.o
 obj-$(CONFIG_SUNVDC)		+= sunvdc.o
 obj-$(CONFIG_BLK_DEV_OSD)	+= osdblk.o
 obj-$(CONFIG_BLK_DEV_UMEM)	+= umem.o
 obj-$(CONFIG_BLK_DEV_NBD)	+= nbd.o
 obj-$(CONFIG_BLK_DEV_CRYPTOLOOP) += cryptoloop.o
 obj-$(CONFIG_VIRTIO_BLK)	+= virtio_blk.o
 obj-$(CONFIG_VIODASD)		+= viodasd.o
 obj-$(CONFIG_BLK_DEV_SX8)	+= sx8.o
 obj-$(CONFIG_BLK_DEV_UB)	+= ub.o
 obj-$(CONFIG_BLK_DEV_HD)	+= hd.o
 obj-$(CONFIG_XEN_BLKDEV_FRONTEND)	+= xen-blkfront.o
 obj-$(CONFIG_XEN_BLKDEV_BACKEND)	+= xen-blkback/
 obj-$(CONFIG_BLK_DEV_DRBD)     += drbd/
 obj-$(CONFIG_BLK_DEV_RBD)     += rbd.o
+obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX)	+= mtip32xx/
 swim_mod-y	:= swim.o swim_asm.o

drivers/block/mtip32xx/Kconfig

Diff comments View file @ 16008d6

File was created	1	#
	2	# mtip32xx device driver configuration
	3	#
	4
	5	config BLK_DEV_PCIESSD_MTIP32XX
	6	tristate "Block Device Driver for Micron PCIe SSDs"
	7	depends on HOTPLUG_PCI_PCIE
	8	help
	9	This enables the block driver for Micron PCIe SSDs.
	10

drivers/block/mtip32xx/Makefile

Diff comments View file @ 16008d6

File was created	1	#
	2	# Makefile for Block device driver for Micron PCIe SSD
	3	#
	4
	5	obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx.o
	6

drivers/block/mtip32xx/mtip32xx.c

Diff comments View file @ 16008d6

File was created	1	/*
	2	* Driver for the Micron P320 SSD
	3	* Copyright (C) 2011 Micron Technology, Inc.
	4	*
	5	* Portions of this code were derived from works subjected to the
	6	* following copyright:
	7	* Copyright (C) 2009 Integrated Device Technology, Inc.
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of the GNU General Public License as published by
	11	* the Free Software Foundation; either version 2 of the License, or
	12	* (at your option) any later version.
	13	*
	14	* This program is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	17	* GNU General Public License for more details.
	18	*
	19	*/
	20
	21	#include <linux/pci.h>
	22	#include <linux/interrupt.h>
	23	#include <linux/ata.h>
	24	#include <linux/delay.h>
	25	#include <linux/hdreg.h>
	26	#include <linux/uaccess.h>
	27	#include <linux/random.h>
	28	#include <linux/smp.h>
	29	#include <linux/compat.h>
	30	#include <linux/fs.h>
	31	#include <linux/module.h>
	32	#include <linux/genhd.h>
	33	#include <linux/blkdev.h>
	34	#include <linux/bio.h>
	35	#include <linux/dma-mapping.h>
	36	#include <linux/idr.h>
	37	#include <linux/kthread.h>
	38	#include <../drivers/ata/ahci.h>
	39	#include "mtip32xx.h"
	40
	41	#define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32)
	42	#define HW_CMD_TBL_SZ (AHCI_CMD_TBL_HDR_SZ + (MTIP_MAX_SG * 16))
	43	#define HW_CMD_TBL_AR_SZ (HW_CMD_TBL_SZ * MTIP_MAX_COMMAND_SLOTS)
	44	#define HW_PORT_PRIV_DMA_SZ \
	45	(HW_CMD_SLOT_SZ + HW_CMD_TBL_AR_SZ + AHCI_RX_FIS_SZ)
	46
	47	#define HOST_HSORG 0xFC
	48	#define HSORG_DISABLE_SLOTGRP_INTR (1<<24)
	49	#define HSORG_DISABLE_SLOTGRP_PXIS (1<<16)
	50	#define HSORG_HWREV 0xFF00
	51	#define HSORG_STYLE 0x8
	52	#define HSORG_SLOTGROUPS 0x7
	53
	54	#define PORT_COMMAND_ISSUE 0x38
	55	#define PORT_SDBV 0x7C
	56
	57	#define PORT_OFFSET 0x100
	58	#define PORT_MEM_SIZE 0x80
	59
	60	#define PORT_IRQ_ERR \
	61	(PORT_IRQ_HBUS_ERR \| PORT_IRQ_IF_ERR \| PORT_IRQ_CONNECT \| \
	62	PORT_IRQ_PHYRDY \| PORT_IRQ_UNK_FIS \| PORT_IRQ_BAD_PMP \| \
	63	PORT_IRQ_TF_ERR \| PORT_IRQ_HBUS_DATA_ERR \| PORT_IRQ_IF_NONFATAL \| \
	64	PORT_IRQ_OVERFLOW)
	65	#define PORT_IRQ_LEGACY \
	66	(PORT_IRQ_PIOS_FIS \| PORT_IRQ_D2H_REG_FIS)
	67	#define PORT_IRQ_HANDLED \
	68	(PORT_IRQ_SDB_FIS \| PORT_IRQ_LEGACY \| \
	69	PORT_IRQ_TF_ERR \| PORT_IRQ_IF_ERR \| \
	70	PORT_IRQ_CONNECT \| PORT_IRQ_PHYRDY)
	71	#define DEF_PORT_IRQ \
	72	(PORT_IRQ_ERR \| PORT_IRQ_LEGACY \| PORT_IRQ_SDB_FIS)
	73
	74	/* product numbers */
	75	#define MTIP_PRODUCT_UNKNOWN 0x00
	76	#define MTIP_PRODUCT_ASICFPGA 0x11
	77
	78	/* Device instance number, incremented each time a device is probed. */
	79	static int instance;
	80
	81	/*
	82	* Global variable used to hold the major block device number
	83	* allocated in mtip_init().
	84	*/
	85	static int mtip_major;
	86
	87	static DEFINE_SPINLOCK(rssd_index_lock);
	88	static DEFINE_IDA(rssd_index_ida);
	89
	90	static int mtip_block_initialize(struct driver_data *dd);
	91
	92	#ifdef CONFIG_COMPAT
	93	struct mtip_compat_ide_task_request_s {
	94	__u8 io_ports[8];
	95	__u8 hob_ports[8];
	96	ide_reg_valid_t out_flags;
	97	ide_reg_valid_t in_flags;
	98	int data_phase;
	99	int req_cmd;
	100	compat_ulong_t out_size;
	101	compat_ulong_t in_size;
	102	};
	103	#endif
	104
	105	/*
	106	* This function check_for_surprise_removal is called
	107	* while card is removed from the system and it will
	108	* read the vendor id from the configration space
	109	*
	110	* @pdev Pointer to the pci_dev structure.
	111	*
	112	* return value
	113	* true if device removed, else false
	114	*/
	115	static bool mtip_check_surprise_removal(struct pci_dev *pdev)
	116	{
	117	u16 vendor_id = 0;
	118
	119	/* Read the vendorID from the configuration space */
	120	pci_read_config_word(pdev, 0x00, &vendor_id);
	121	if (vendor_id == 0xFFFF)
	122	return true; /* device removed */
	123
	124	return false; /* device present */
	125	}
	126
	127	/*
	128	* This function is called for clean the pending command in the
	129	* command slot during the surprise removal of device and return
	130	* error to the upper layer.
	131	*
	132	* @dd Pointer to the DRIVER_DATA structure.
	133	*
	134	* return value
	135	* None
	136	*/
	137	static void mtip_command_cleanup(struct driver_data *dd)
	138	{
	139	int group = 0, commandslot = 0, commandindex = 0;
	140	struct mtip_cmd *command;
	141	struct mtip_port *port = dd->port;
	142
	143	for (group = 0; group < 4; group++) {
	144	for (commandslot = 0; commandslot < 32; commandslot++) {
	145	if (!(port->allocated[group] & (1 << commandslot)))
	146	continue;
	147
	148	commandindex = group << 5 \| commandslot;
	149	command = &port->commands[commandindex];
	150
	151	if (atomic_read(&command->active)
	152	&& (command->async_callback)) {
	153	command->async_callback(command->async_data,
	154	-ENODEV);
	155	command->async_callback = NULL;
	156	command->async_data = NULL;
	157	}
	158
	159	dma_unmap_sg(&port->dd->pdev->dev,
	160	command->sg,
	161	command->scatter_ents,
	162	command->direction);
	163	}
	164	}
	165
	166	up(&port->cmd_slot);
	167
	168	atomic_set(&dd->drv_cleanup_done, true);
	169	}
	170
	171	/*
	172	* Obtain an empty command slot.
	173	*
	174	* This function needs to be reentrant since it could be called
	175	* at the same time on multiple CPUs. The allocation of the
	176	* command slot must be atomic.
	177	*
	178	* @port Pointer to the port data structure.
	179	*
	180	* return value
	181	* >= 0 Index of command slot obtained.
	182	* -1 No command slots available.
	183	*/
	184	static int get_slot(struct mtip_port *port)
	185	{
	186	int slot, i;
	187	unsigned int num_command_slots = port->dd->slot_groups * 32;
	188
	189	/*
	190	* Try 10 times, because there is a small race here.
	191	* that's ok, because it's still cheaper than a lock.
	192	*
	193	* Race: Since this section is not protected by lock, same bit
	194	* could be chosen by different process contexts running in
	195	* different processor. So instead of costly lock, we are going
	196	* with loop.
	197	*/
	198	for (i = 0; i < 10; i++) {
	199	slot = find_next_zero_bit(port->allocated,
	200	num_command_slots, 1);
	201	if ((slot < num_command_slots) &&
	202	(!test_and_set_bit(slot, port->allocated)))
	203	return slot;
	204	}
	205	dev_warn(&port->dd->pdev->dev, "Failed to get a tag.\n");
	206
	207	if (mtip_check_surprise_removal(port->dd->pdev)) {
	208	/* Device not present, clean outstanding commands */
	209	mtip_command_cleanup(port->dd);
	210	}
	211	return -1;
	212	}
	213
	214	/*
	215	* Release a command slot.
	216	*
	217	* @port Pointer to the port data structure.
	218	* @tag Tag of command to release
	219	*
	220	* return value
	221	* None
	222	*/
	223	static inline void release_slot(struct mtip_port *port, int tag)
	224	{
	225	smp_mb__before_clear_bit();
	226	clear_bit(tag, port->allocated);
	227	smp_mb__after_clear_bit();
	228	}
	229
	230	/*
	231	* Reset the HBA (without sleeping)
	232	*
	233	* Just like hba_reset, except does not call sleep, so can be
	234	* run from interrupt/tasklet context.
	235	*
	236	* @dd Pointer to the driver data structure.
	237	*
	238	* return value
	239	* 0 The reset was successful.
	240	* -1 The HBA Reset bit did not clear.
	241	*/
	242	static int hba_reset_nosleep(struct driver_data *dd)
	243	{
	244	unsigned long timeout;
	245
	246	/* Chip quirk: quiesce any chip function */
	247	mdelay(10);
	248
	249	/* Set the reset bit */
	250	writel(HOST_RESET, dd->mmio + HOST_CTL);
	251
	252	/* Flush */
	253	readl(dd->mmio + HOST_CTL);
	254
	255	/*
	256	* Wait 10ms then spin for up to 1 second
	257	* waiting for reset acknowledgement
	258	*/
	259	timeout = jiffies + msecs_to_jiffies(1000);
	260	mdelay(10);
	261	while ((readl(dd->mmio + HOST_CTL) & HOST_RESET)
	262	&& time_before(jiffies, timeout))
	263	mdelay(1);
	264
	265	if (readl(dd->mmio + HOST_CTL) & HOST_RESET)
	266	return -1;
	267
	268	return 0;
	269	}
	270
	271	/*
	272	* Issue a command to the hardware.
	273	*
	274	* Set the appropriate bit in the s_active and Command Issue hardware
	275	* registers, causing hardware command processing to begin.
	276	*
	277	* @port Pointer to the port structure.
	278	* @tag The tag of the command to be issued.
	279	*
	280	* return value
	281	* None
	282	*/
	283	static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag)
	284	{
	285	unsigned long flags = 0;
	286
	287	atomic_set(&port->commands[tag].active, 1);
	288
	289	spin_lock_irqsave(&port->cmd_issue_lock, flags);
	290
	291	writel((1 << MTIP_TAG_BIT(tag)),
	292	port->s_active[MTIP_TAG_INDEX(tag)]);
	293	writel((1 << MTIP_TAG_BIT(tag)),
	294	port->cmd_issue[MTIP_TAG_INDEX(tag)]);
	295
	296	spin_unlock_irqrestore(&port->cmd_issue_lock, flags);
	297	}
	298
	299	/*
	300	* Enable/disable the reception of FIS
	301	*
	302	* @port Pointer to the port data structure
	303	* @enable 1 to enable, 0 to disable
	304	*
	305	* return value
	306	* Previous state: 1 enabled, 0 disabled
	307	*/
	308	static int mtip_enable_fis(struct mtip_port *port, int enable)
	309	{
	310	u32 tmp;
	311
	312	/* enable FIS reception */
	313	tmp = readl(port->mmio + PORT_CMD);
	314	if (enable)
	315	writel(tmp \| PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
	316	else
	317	writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
	318
	319	/* Flush */
	320	readl(port->mmio + PORT_CMD);
	321
	322	return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX));
	323	}
	324
	325	/*
	326	* Enable/disable the DMA engine
	327	*
	328	* @port Pointer to the port data structure
	329	* @enable 1 to enable, 0 to disable
	330	*
	331	* return value
	332	* Previous state: 1 enabled, 0 disabled.
	333	*/
	334	static int mtip_enable_engine(struct mtip_port *port, int enable)
	335	{
	336	u32 tmp;
	337
	338	/* enable FIS reception */
	339	tmp = readl(port->mmio + PORT_CMD);
	340	if (enable)
	341	writel(tmp \| PORT_CMD_START, port->mmio + PORT_CMD);
	342	else
	343	writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD);
	344
	345	readl(port->mmio + PORT_CMD);
	346	return (((tmp & PORT_CMD_START) == PORT_CMD_START));
	347	}
	348
	349	/*
	350	* Enables the port DMA engine and FIS reception.
	351	*
	352	* return value
	353	* None
	354	*/
	355	static inline void mtip_start_port(struct mtip_port *port)
	356	{
	357	/* Enable FIS reception */
	358	mtip_enable_fis(port, 1);
	359
	360	/* Enable the DMA engine */
	361	mtip_enable_engine(port, 1);
	362	}
	363
	364	/*
	365	* Deinitialize a port by disabling port interrupts, the DMA engine,
	366	* and FIS reception.
	367	*
	368	* @port Pointer to the port structure
	369	*
	370	* return value
	371	* None
	372	*/
	373	static inline void mtip_deinit_port(struct mtip_port *port)
	374	{
	375	/* Disable interrupts on this port */
	376	writel(0, port->mmio + PORT_IRQ_MASK);
	377
	378	/* Disable the DMA engine */
	379	mtip_enable_engine(port, 0);
	380
	381	/* Disable FIS reception */
	382	mtip_enable_fis(port, 0);
	383	}
	384
	385	/*
	386	* Initialize a port.
	387	*
	388	* This function deinitializes the port by calling mtip_deinit_port() and
	389	* then initializes it by setting the command header and RX FIS addresses,
	390	* clearing the SError register and any pending port interrupts before
	391	* re-enabling the default set of port interrupts.
	392	*
	393	* @port Pointer to the port structure.
	394	*
	395	* return value
	396	* None
	397	*/
	398	static void mtip_init_port(struct mtip_port *port)
	399	{
	400	int i;
	401	mtip_deinit_port(port);
	402
	403	/* Program the command list base and FIS base addresses */
	404	if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) {
	405	writel((port->command_list_dma >> 16) >> 16,
	406	port->mmio + PORT_LST_ADDR_HI);
	407	writel((port->rxfis_dma >> 16) >> 16,
	408	port->mmio + PORT_FIS_ADDR_HI);
	409	}
	410
	411	writel(port->command_list_dma & 0xFFFFFFFF,
	412	port->mmio + PORT_LST_ADDR);
	413	writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR);
	414
	415	/* Clear SError */
	416	writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
	417
	418	/* reset the completed registers.*/
	419	for (i = 0; i < port->dd->slot_groups; i++)
	420	writel(0xFFFFFFFF, port->completed[i]);
	421
	422	/* Clear any pending interrupts for this port */
	423	writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT);
	424
	425	/* Enable port interrupts */
	426	writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK);
	427	}
	428
	429	/*
	430	* Restart a port
	431	*
	432	* @port Pointer to the port data structure.
	433	*
	434	* return value
	435	* None
	436	*/
	437	static void mtip_restart_port(struct mtip_port *port)
	438	{
	439	unsigned long timeout;
	440
	441	/* Disable the DMA engine */
	442	mtip_enable_engine(port, 0);
	443
	444	/* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */
	445	timeout = jiffies + msecs_to_jiffies(500);
	446	while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON)
	447	&& time_before(jiffies, timeout))
	448	;
	449
	450	/*
	451	* Chip quirk: escalate to hba reset if
	452	* PxCMD.CR not clear after 500 ms
	453	*/
	454	if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) {
	455	dev_warn(&port->dd->pdev->dev,
	456	"PxCMD.CR not clear, escalating reset\n");
	457
	458	if (hba_reset_nosleep(port->dd))
	459	dev_err(&port->dd->pdev->dev,
	460	"HBA reset escalation failed.\n");
	461
	462	/* 30 ms delay before com reset to quiesce chip */
	463	mdelay(30);
	464	}
	465
	466	dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n");
	467
	468	/* Set PxSCTL.DET */
	469	writel(readl(port->mmio + PORT_SCR_CTL) \|
	470	1, port->mmio + PORT_SCR_CTL);
	471	readl(port->mmio + PORT_SCR_CTL);
	472
	473	/* Wait 1 ms to quiesce chip function */
	474	timeout = jiffies + msecs_to_jiffies(1);
	475	while (time_before(jiffies, timeout))
	476	;
	477
	478	/* Clear PxSCTL.DET */
	479	writel(readl(port->mmio + PORT_SCR_CTL) & ~1,
	480	port->mmio + PORT_SCR_CTL);
	481	readl(port->mmio + PORT_SCR_CTL);
	482
	483	/* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */
	484	timeout = jiffies + msecs_to_jiffies(500);
	485	while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
	486	&& time_before(jiffies, timeout))
	487	;
	488
	489	if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
	490	dev_warn(&port->dd->pdev->dev,
	491	"COM reset failed\n");
	492
	493	/* Clear SError, the PxSERR.DIAG.x should be set so clear it */
	494	writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
	495
	496	/* Enable the DMA engine */
	497	mtip_enable_engine(port, 1);
	498	}
	499
	500	/*
	501	* Called periodically to see if any read/write commands are
	502	* taking too long to complete.
	503	*
	504	* @data Pointer to the PORT data structure.
	505	*
	506	* return value
	507	* None
	508	*/
	509	static void mtip_timeout_function(unsigned long int data)
	510	{
	511	struct mtip_port port = (struct mtip_port ) data;
	512	struct host_to_dev_fis *fis;
	513	struct mtip_cmd *command;
	514	int tag, cmdto_cnt = 0;
	515	unsigned int bit, group;
	516	unsigned int num_command_slots = port->dd->slot_groups * 32;
	517
	518	if (unlikely(!port))
	519	return;
	520
	521	if (atomic_read(&port->dd->resumeflag) == true) {
	522	mod_timer(&port->cmd_timer,
	523	jiffies + msecs_to_jiffies(30000));
	524	return;
	525	}
	526
	527	for (tag = 0; tag < num_command_slots; tag++) {
	528	/*
	529	* Skip internal command slot as it has
	530	* its own timeout mechanism
	531	*/
	532	if (tag == MTIP_TAG_INTERNAL)
	533	continue;
	534
	535	if (atomic_read(&port->commands[tag].active) &&
	536	(time_after(jiffies, port->commands[tag].comp_time))) {
	537	group = tag >> 5;
	538	bit = tag & 0x1F;
	539
	540	command = &port->commands[tag];
	541	fis = (struct host_to_dev_fis *) command->command;
	542
	543	dev_warn(&port->dd->pdev->dev,
	544	"Timeout for command tag %d\n", tag);
	545
	546	cmdto_cnt++;
	547	if (cmdto_cnt == 1)
	548	set_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags);
	549
	550	/*
	551	* Clear the completed bit. This should prevent
	552	* any interrupt handlers from trying to retire
	553	* the command.
	554	*/
	555	writel(1 << bit, port->completed[group]);
	556
	557	/* Call the async completion callback. */
	558	if (likely(command->async_callback))
	559	command->async_callback(command->async_data,
	560	-EIO);
	561	command->async_callback = NULL;
	562	command->comp_func = NULL;
	563
	564	/* Unmap the DMA scatter list entries */
	565	dma_unmap_sg(&port->dd->pdev->dev,
	566	command->sg,
	567	command->scatter_ents,
	568	command->direction);
	569
	570	/*
	571	* Clear the allocated bit and active tag for the
	572	* command.
	573	*/
	574	atomic_set(&port->commands[tag].active, 0);
	575	release_slot(port, tag);
	576
	577	up(&port->cmd_slot);
	578	}
	579	}
	580
	581	if (cmdto_cnt) {
	582	dev_warn(&port->dd->pdev->dev,
	583	"%d commands timed out: restarting port",
	584	cmdto_cnt);
	585	mtip_restart_port(port);
	586	clear_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags);
	587	wake_up_interruptible(&port->svc_wait);
	588	}
	589
	590	/* Restart the timer */
	591	mod_timer(&port->cmd_timer,
	592	jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD));
	593	}
	594
	595	/*
	596	* IO completion function.
	597	*
	598	* This completion function is called by the driver ISR when a
	599	* command that was issued by the kernel completes. It first calls the
	600	* asynchronous completion function which normally calls back into the block
	601	* layer passing the asynchronous callback data, then unmaps the
	602	* scatter list associated with the completed command, and finally
	603	* clears the allocated bit associated with the completed command.
	604	*
	605	* @port Pointer to the port data structure.
	606	* @tag Tag of the command.
	607	* @data Pointer to driver_data.
	608	* @status Completion status.
	609	*
	610	* return value
	611	* None
	612	*/
	613	static void mtip_async_complete(struct mtip_port *port,
	614	int tag,
	615	void *data,
	616	int status)
	617	{
	618	struct mtip_cmd *command;
	619	struct driver_data *dd = data;
	620	int cb_status = status ? -EIO : 0;
	621
	622	if (unlikely(!dd) \|\| unlikely(!port))
	623	return;
	624
	625	command = &port->commands[tag];
	626
	627	if (unlikely(status == PORT_IRQ_TF_ERR)) {
	628	dev_warn(&port->dd->pdev->dev,
	629	"Command tag %d failed due to TFE\n", tag);
	630	}
	631
	632	/* Upper layer callback */
	633	if (likely(command->async_callback))
	634	command->async_callback(command->async_data, cb_status);
	635
	636	command->async_callback = NULL;
	637	command->comp_func = NULL;
	638
	639	/* Unmap the DMA scatter list entries */
	640	dma_unmap_sg(&dd->pdev->dev,
	641	command->sg,
	642	command->scatter_ents,
	643	command->direction);
	644
	645	/* Clear the allocated and active bits for the command */
	646	atomic_set(&port->commands[tag].active, 0);
	647	release_slot(port, tag);
	648
	649	up(&port->cmd_slot);
	650	}
	651
	652	/*
	653	* Internal command completion callback function.
	654	*
	655	* This function is normally called by the driver ISR when an internal
	656	* command completed. This function signals the command completion by
	657	* calling complete().
	658	*
	659	* @port Pointer to the port data structure.
	660	* @tag Tag of the command that has completed.
	661	* @data Pointer to a completion structure.
	662	* @status Completion status.
	663	*
	664	* return value
	665	* None
	666	*/
	667	static void mtip_completion(struct mtip_port *port,
	668	int tag,
	669	void *data,
	670	int status)
	671	{
	672	struct mtip_cmd *command = &port->commands[tag];
	673	struct completion *waiting = data;
	674	if (unlikely(status == PORT_IRQ_TF_ERR))
	675	dev_warn(&port->dd->pdev->dev,
	676	"Internal command %d completed with TFE\n", tag);
	677
	678	command->async_callback = NULL;
	679	command->comp_func = NULL;
	680
	681	complete(waiting);
	682	}
	683
	684	/*
	685	* Helper function for tag logging
	686	*/
	687	static void print_tags(struct driver_data *dd,
	688	char *msg,
	689	unsigned long *tagbits)
	690	{
	691	unsigned int tag, count = 0;
	692
	693	for (tag = 0; tag < (dd->slot_groups) * 32; tag++) {
	694	if (test_bit(tag, tagbits))
	695	count++;
	696	}
	697	if (count)
	698	dev_info(&dd->pdev->dev, "%s [%i tags]\n", msg, count);
	699	}
	700
	701	/*
	702	* Handle an error.
	703	*
	704	* @dd Pointer to the DRIVER_DATA structure.
	705	*
	706	* return value
	707	* None
	708	*/
	709	static void mtip_handle_tfe(struct driver_data *dd)
	710	{
	711	int group, tag, bit, reissue;
	712	struct mtip_port *port;
	713	struct mtip_cmd *command;
	714	u32 completed;
	715	struct host_to_dev_fis *fis;
	716	unsigned long tagaccum[SLOTBITS_IN_LONGS];
	717
	718	dev_warn(&dd->pdev->dev, "Taskfile error\n");
	719
	720	port = dd->port;
	721
	722	/* Stop the timer to prevent command timeouts. */
	723	del_timer(&port->cmd_timer);
	724
	725	/* Set eh_active */
	726	set_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags);
	727
	728	/* Loop through all the groups */
	729	for (group = 0; group < dd->slot_groups; group++) {
	730	completed = readl(port->completed[group]);
	731
	732	/* clear completed status register in the hardware.*/
	733	writel(completed, port->completed[group]);
	734
	735	/* clear the tag accumulator */
	736	memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
	737
	738	/* Process successfully completed commands */
	739	for (bit = 0; bit < 32 && completed; bit++) {
	740	if (!(completed & (1<<bit)))
	741	continue;
	742	tag = (group << 5) + bit;
	743
	744	/* Skip the internal command slot */
	745	if (tag == MTIP_TAG_INTERNAL)
	746	continue;
	747
	748	command = &port->commands[tag];
	749	if (likely(command->comp_func)) {
	750	set_bit(tag, tagaccum);
	751	atomic_set(&port->commands[tag].active, 0);
	752	command->comp_func(port,
	753	tag,
	754	command->comp_data,
	755	0);
	756	} else {
	757	dev_err(&port->dd->pdev->dev,
	758	"Missing completion func for tag %d",
	759	tag);
	760	if (mtip_check_surprise_removal(dd->pdev)) {
	761	mtip_command_cleanup(dd);
	762	/* don't proceed further */
	763	return;
	764	}
	765	}
	766	}
	767	}
	768	print_tags(dd, "TFE tags completed:", tagaccum);
	769
	770	/* Restart the port */
	771	mdelay(20);
	772	mtip_restart_port(port);
	773
	774	/* clear the tag accumulator */
	775	memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
	776
	777	/* Loop through all the groups */
	778	for (group = 0; group < dd->slot_groups; group++) {
	779	for (bit = 0; bit < 32; bit++) {
	780	reissue = 1;
	781	tag = (group << 5) + bit;
	782
	783	/* If the active bit is set re-issue the command */
	784	if (atomic_read(&port->commands[tag].active) == 0)
	785	continue;
	786
	787	fis = (struct host_to_dev_fis *)
	788	port->commands[tag].command;
	789
	790	/* Should re-issue? */
	791	if (tag == MTIP_TAG_INTERNAL \|\|
	792	fis->command == ATA_CMD_SET_FEATURES)
	793	reissue = 0;
	794
	795	/*
	796	* First check if this command has
	797	* exceeded its retries.
	798	*/
	799	if (reissue &&
	800	(port->commands[tag].retries-- > 0)) {
	801
	802	set_bit(tag, tagaccum);
	803
	804	/* Update the timeout value. */
	805	port->commands[tag].comp_time =
	806	jiffies + msecs_to_jiffies(
	807	MTIP_NCQ_COMMAND_TIMEOUT_MS);
	808	/* Re-issue the command. */
	809	mtip_issue_ncq_command(port, tag);
	810
	811	continue;
	812	}
	813
	814	/* Retire a command that will not be reissued */
	815	dev_warn(&port->dd->pdev->dev,
	816	"retiring tag %d\n", tag);
	817	atomic_set(&port->commands[tag].active, 0);
	818
	819	if (port->commands[tag].comp_func)
	820	port->commands[tag].comp_func(
	821	port,
	822	tag,
	823	port->commands[tag].comp_data,
	824	PORT_IRQ_TF_ERR);
	825	else
	826	dev_warn(&port->dd->pdev->dev,
	827	"Bad completion for tag %d\n",
	828	tag);
	829	}
	830	}
	831	print_tags(dd, "TFE tags reissued:", tagaccum);
	832
	833	/* clear eh_active */
	834	clear_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags);
	835	wake_up_interruptible(&port->svc_wait);
	836
	837	mod_timer(&port->cmd_timer,
	838	jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD));
	839	}
	840
	841	/*
	842	* Handle a set device bits interrupt
	843	*/
	844	static inline void mtip_process_sdbf(struct driver_data *dd)
	845	{
	846	struct mtip_port *port = dd->port;
	847	int group, tag, bit;
	848	u32 completed;
	849	struct mtip_cmd *command;
	850
	851	/* walk all bits in all slot groups */
	852	for (group = 0; group < dd->slot_groups; group++) {
	853	completed = readl(port->completed[group]);
	854
	855	/* clear completed status register in the hardware.*/
	856	writel(completed, port->completed[group]);
	857
	858	/* Process completed commands. */
	859	for (bit = 0;
	860	(bit < 32) && completed;
	861	bit++, completed >>= 1) {
	862	if (completed & 0x01) {
	863	tag = (group << 5) \| bit;
	864
	865	/* skip internal command slot. */
	866	if (unlikely(tag == MTIP_TAG_INTERNAL))
	867	continue;
	868
	869	command = &port->commands[tag];
	870	/* make internal callback */
	871	if (likely(command->comp_func)) {
	872	command->comp_func(
	873	port,
	874	tag,
	875	command->comp_data,
	876	0);
	877	} else {
	878	dev_warn(&dd->pdev->dev,
	879	"Null completion "
	880	"for tag %d",
	881	tag);
	882
	883	if (mtip_check_surprise_removal(
	884	dd->pdev)) {
	885	mtip_command_cleanup(dd);
	886	return;
	887	}
	888	}
	889	}
	890	}
	891	}
	892	}
	893
	894	/*
	895	* Process legacy pio and d2h interrupts
	896	*/
	897	static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat)
	898	{
	899	struct mtip_port *port = dd->port;
	900	struct mtip_cmd *cmd = &port->commands[MTIP_TAG_INTERNAL];
	901
	902	if (test_bit(MTIP_FLAG_IC_ACTIVE_BIT, &port->flags) &&
	903	(cmd != NULL) && !(readl(port->cmd_issue[MTIP_TAG_INTERNAL])
	904	& (1 << MTIP_TAG_INTERNAL))) {
	905	if (cmd->comp_func) {
	906	cmd->comp_func(port,
	907	MTIP_TAG_INTERNAL,
	908	cmd->comp_data,
	909	0);
	910	return;
	911	}
	912	}
	913
	914	dev_warn(&dd->pdev->dev, "IRQ status 0x%x ignored.\n", port_stat);
	915
	916	return;
	917	}
	918
	919	/*
	920	* Demux and handle errors
	921	*/
	922	static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat)
	923	{
	924	if (likely(port_stat & (PORT_IRQ_TF_ERR \| PORT_IRQ_IF_ERR)))
	925	mtip_handle_tfe(dd);
	926
	927	if (unlikely(port_stat & PORT_IRQ_CONNECT)) {
	928	dev_warn(&dd->pdev->dev,
	929	"Clearing PxSERR.DIAG.x\n");
	930	writel((1 << 26), dd->port->mmio + PORT_SCR_ERR);
	931	}
	932
	933	if (unlikely(port_stat & PORT_IRQ_PHYRDY)) {
	934	dev_warn(&dd->pdev->dev,
	935	"Clearing PxSERR.DIAG.n\n");
	936	writel((1 << 16), dd->port->mmio + PORT_SCR_ERR);
	937	}
	938
	939	if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) {
	940	dev_warn(&dd->pdev->dev,
	941	"Port stat errors %x unhandled\n",
	942	(port_stat & ~PORT_IRQ_HANDLED));
	943	}
	944	}
	945
	946	static inline irqreturn_t mtip_handle_irq(struct driver_data *data)
	947	{
	948	struct driver_data dd = (struct driver_data ) data;
	949	struct mtip_port *port = dd->port;
	950	u32 hba_stat, port_stat;
	951	int rv = IRQ_NONE;
	952
	953	hba_stat = readl(dd->mmio + HOST_IRQ_STAT);
	954	if (hba_stat) {
	955	rv = IRQ_HANDLED;
	956
	957	/* Acknowledge the interrupt status on the port.*/
	958	port_stat = readl(port->mmio + PORT_IRQ_STAT);
	959	writel(port_stat, port->mmio + PORT_IRQ_STAT);
	960
	961	/* Demux port status */
	962	if (likely(port_stat & PORT_IRQ_SDB_FIS))
	963	mtip_process_sdbf(dd);
	964
	965	if (unlikely(port_stat & PORT_IRQ_ERR)) {
	966	if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
	967	mtip_command_cleanup(dd);
	968	/* don't proceed further */
	969	return IRQ_HANDLED;
	970	}
	971
	972	mtip_process_errors(dd, port_stat & PORT_IRQ_ERR);
	973	}
	974
	975	if (unlikely(port_stat & PORT_IRQ_LEGACY))
	976	mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY);
	977	}
	978
	979	/* acknowledge interrupt */
	980	writel(hba_stat, dd->mmio + HOST_IRQ_STAT);
	981
	982	return rv;
	983	}
	984
	985	/*
	986	* Wrapper for mtip_handle_irq
	987	* (ignores return code)
	988	*/
	989	static void mtip_tasklet(unsigned long data)
	990	{
	991	mtip_handle_irq((struct driver_data *) data);
	992	}
	993
	994	/*
	995	* HBA interrupt subroutine.
	996	*
	997	* @irq IRQ number.
	998	* @instance Pointer to the driver data structure.
	999	*
	1000	* return value
	1001	* IRQ_HANDLED A HBA interrupt was pending and handled.
	1002	* IRQ_NONE This interrupt was not for the HBA.
	1003	*/
	1004	static irqreturn_t mtip_irq_handler(int irq, void *instance)
	1005	{
	1006	struct driver_data *dd = instance;
	1007	tasklet_schedule(&dd->tasklet);
	1008	return IRQ_HANDLED;
	1009	}
	1010
	1011	static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag)
	1012	{
	1013	atomic_set(&port->commands[tag].active, 1);
	1014	writel(1 << MTIP_TAG_BIT(tag),
	1015	port->cmd_issue[MTIP_TAG_INDEX(tag)]);
	1016	}
	1017
	1018	/*
	1019	* Wait for port to quiesce
	1020	*
	1021	* @port Pointer to port data structure
	1022	* @timeout Max duration to wait (ms)
	1023	*
	1024	* return value
	1025	* 0 Success
	1026	* -EBUSY Commands still active
	1027	*/
	1028	static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout)
	1029	{
	1030	unsigned long to;
	1031	unsigned int n;
	1032	unsigned int active = 1;
	1033
	1034	to = jiffies + msecs_to_jiffies(timeout);
	1035	do {
	1036	if (test_bit(MTIP_FLAG_SVC_THD_ACTIVE_BIT, &port->flags) &&
	1037	test_bit(MTIP_FLAG_ISSUE_CMDS_BIT, &port->flags)) {
	1038	msleep(20);
	1039	continue; /* svc thd is actively issuing commands */
	1040	}
	1041	/*
	1042	* Ignore s_active bit 0 of array element 0.
	1043	* This bit will always be set
	1044	*/
	1045	active = readl(port->s_active[0]) & 0xFFFFFFFE;
	1046	for (n = 1; n < port->dd->slot_groups; n++)
	1047	active \|= readl(port->s_active[n]);
	1048
	1049	if (!active)
	1050	break;
	1051
	1052	msleep(20);
	1053	} while (time_before(jiffies, to));
	1054
	1055	return active ? -EBUSY : 0;
	1056	}
	1057
	1058	/*
	1059	* Execute an internal command and wait for the completion.
	1060	*
	1061	* @port Pointer to the port data structure.
	1062	* @fis Pointer to the FIS that describes the command.
	1063	* @fis_len Length in WORDS of the FIS.
	1064	* @buffer DMA accessible for command data.
	1065	* @buf_len Length, in bytes, of the data buffer.
	1066	* @opts Command header options, excluding the FIS length
	1067	* and the number of PRD entries.
	1068	* @timeout Time in ms to wait for the command to complete.
	1069	*
	1070	* return value
	1071	* 0 Command completed successfully.
	1072	* -EFAULT The buffer address is not correctly aligned.
	1073	* -EBUSY Internal command or other IO in progress.
	1074	* -EAGAIN Time out waiting for command to complete.
	1075	*/
	1076	static int mtip_exec_internal_command(struct mtip_port *port,
	1077	void *fis,
	1078	int fis_len,
	1079	dma_addr_t buffer,
	1080	int buf_len,
	1081	u32 opts,
	1082	gfp_t atomic,
	1083	unsigned long timeout)
	1084	{
	1085	struct mtip_cmd_sg *command_sg;
	1086	DECLARE_COMPLETION_ONSTACK(wait);
	1087	int rv = 0;
	1088	struct mtip_cmd *int_cmd = &port->commands[MTIP_TAG_INTERNAL];
	1089
	1090	/* Make sure the buffer is 8 byte aligned. This is asic specific. */
	1091	if (buffer & 0x00000007) {
	1092	dev_err(&port->dd->pdev->dev,
	1093	"SG buffer is not 8 byte aligned\n");
	1094	return -EFAULT;
	1095	}
	1096
	1097	/* Only one internal command should be running at a time */
	1098	if (test_and_set_bit(MTIP_TAG_INTERNAL, port->allocated)) {
	1099	dev_warn(&port->dd->pdev->dev,
	1100	"Internal command already active\n");
	1101	return -EBUSY;
	1102	}
	1103	set_bit(MTIP_FLAG_IC_ACTIVE_BIT, &port->flags);
	1104
	1105	if (atomic == GFP_KERNEL) {
	1106	/* wait for io to complete if non atomic */
	1107	if (mtip_quiesce_io(port, 5000) < 0) {
	1108	dev_warn(&port->dd->pdev->dev,
	1109	"Failed to quiesce IO\n");
	1110	release_slot(port, MTIP_TAG_INTERNAL);
	1111	clear_bit(MTIP_FLAG_IC_ACTIVE_BIT, &port->flags);
	1112	wake_up_interruptible(&port->svc_wait);
	1113	return -EBUSY;
	1114	}
	1115
	1116	/* Set the completion function and data for the command. */
	1117	int_cmd->comp_data = &wait;
	1118	int_cmd->comp_func = mtip_completion;
	1119
	1120	} else {
	1121	/* Clear completion - we're going to poll */
	1122	int_cmd->comp_data = NULL;
	1123	int_cmd->comp_func = NULL;
	1124	}
	1125
	1126	/* Copy the command to the command table */
	1127	memcpy(int_cmd->command, fis, fis_len*4);
	1128
	1129	/* Populate the SG list */
	1130	int_cmd->command_header->opts =
	1131	__force_bit2int cpu_to_le32(opts \| fis_len);
	1132	if (buf_len) {
	1133	command_sg = int_cmd->command + AHCI_CMD_TBL_HDR_SZ;
	1134
	1135	command_sg->info =
	1136	__force_bit2int cpu_to_le32((buf_len-1) & 0x3FFFFF);
	1137	command_sg->dba =
	1138	__force_bit2int cpu_to_le32(buffer & 0xFFFFFFFF);
	1139	command_sg->dba_upper =
	1140	__force_bit2int cpu_to_le32((buffer >> 16) >> 16);
	1141
	1142	int_cmd->command_header->opts \|=
	1143	__force_bit2int cpu_to_le32((1 << 16));
	1144	}
	1145
	1146	/* Populate the command header */
	1147	int_cmd->command_header->byte_count = 0;
	1148
	1149	/* Issue the command to the hardware */
	1150	mtip_issue_non_ncq_command(port, MTIP_TAG_INTERNAL);
	1151
	1152	/* Poll if atomic, wait_for_completion otherwise */
	1153	if (atomic == GFP_KERNEL) {
	1154	/* Wait for the command to complete or timeout. */
	1155	if (wait_for_completion_timeout(
	1156	&wait,
	1157	msecs_to_jiffies(timeout)) == 0) {
	1158	dev_err(&port->dd->pdev->dev,
	1159	"Internal command did not complete [%d] "
	1160	"within timeout of %lu ms\n",
	1161	atomic, timeout);
	1162	rv = -EAGAIN;
	1163	}
	1164
	1165	if (readl(port->cmd_issue[MTIP_TAG_INTERNAL])
	1166	& (1 << MTIP_TAG_INTERNAL)) {
	1167	dev_warn(&port->dd->pdev->dev,
	1168	"Retiring internal command but CI is 1.\n");
	1169	}
	1170
	1171	} else {
	1172	/* Spin for <timeout> checking if command still outstanding */
	1173	timeout = jiffies + msecs_to_jiffies(timeout);
	1174
	1175	while ((readl(
	1176	port->cmd_issue[MTIP_TAG_INTERNAL])
	1177	& (1 << MTIP_TAG_INTERNAL))
	1178	&& time_before(jiffies, timeout))
	1179	;
	1180
	1181	if (readl(port->cmd_issue[MTIP_TAG_INTERNAL])
	1182	& (1 << MTIP_TAG_INTERNAL)) {
	1183	dev_err(&port->dd->pdev->dev,
	1184	"Internal command did not complete [%d]\n",
	1185	atomic);
	1186	rv = -EAGAIN;
	1187	}
	1188	}
	1189
	1190	/* Clear the allocated and active bits for the internal command. */
	1191	atomic_set(&int_cmd->active, 0);
	1192	release_slot(port, MTIP_TAG_INTERNAL);
	1193	clear_bit(MTIP_FLAG_IC_ACTIVE_BIT, &port->flags);
	1194	wake_up_interruptible(&port->svc_wait);
	1195
	1196	return rv;
	1197	}
	1198
	1199	/*
	1200	* Byte-swap ATA ID strings.
	1201	*
	1202	* ATA identify data contains strings in byte-swapped 16-bit words.
	1203	* They must be swapped (on all architectures) to be usable as C strings.
	1204	* This function swaps bytes in-place.
	1205	*
	1206	* @buf The buffer location of the string
	1207	* @len The number of bytes to swap
	1208	*
	1209	* return value
	1210	* None
	1211	*/
	1212	static inline void ata_swap_string(u16 *buf, unsigned int len)
	1213	{
	1214	int i;
	1215	for (i = 0; i < (len/2); i++)
	1216	be16_to_cpus(&buf[i]);
	1217	}
	1218
	1219	/*
	1220	* Request the device identity information.
	1221	*
	1222	* If a user space buffer is not specified, i.e. is NULL, the
	1223	* identify information is still read from the drive and placed
	1224	* into the identify data buffer (@e port->identify) in the
	1225	* port data structure.
	1226	* When the identify buffer contains valid identify information @e
	1227	* port->identify_valid is non-zero.
	1228	*
	1229	* @port Pointer to the port structure.
	1230	* @user_buffer A user space buffer where the identify data should be
	1231	* copied.
	1232	*
	1233	* return value
	1234	* 0 Command completed successfully.
	1235	* -EFAULT An error occurred while coping data to the user buffer.
	1236	* -1 Command failed.
	1237	*/
	1238	static int mtip_get_identify(struct mtip_port port, void __user user_buffer)
	1239	{
	1240	int rv = 0;
	1241	struct host_to_dev_fis fis;
	1242
	1243	/* Build the FIS. */
	1244	memset(&fis, 0, sizeof(struct host_to_dev_fis));
	1245	fis.type = 0x27;
	1246	fis.opts = 1 << 7;
	1247	fis.command = ATA_CMD_ID_ATA;
	1248
	1249	/* Set the identify information as invalid. */
	1250	port->identify_valid = 0;
	1251
	1252	/* Clear the identify information. */
	1253	memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS);
	1254
	1255	/* Execute the command. */
	1256	if (mtip_exec_internal_command(port,
	1257	&fis,
	1258	5,
	1259	port->identify_dma,
	1260	sizeof(u16) * ATA_ID_WORDS,
	1261	0,
	1262	GFP_KERNEL,
	1263	MTIP_INTERNAL_COMMAND_TIMEOUT_MS)
	1264	< 0) {
	1265	rv = -1;
	1266	goto out;
	1267	}
	1268
	1269	/*
	1270	* Perform any necessary byte-swapping. Yes, the kernel does in fact
	1271	* perform field-sensitive swapping on the string fields.
	1272	* See the kernel use of ata_id_string() for proof of this.
	1273	*/
	1274	#ifdef __LITTLE_ENDIAN
	1275	ata_swap_string(port->identify + 27, 40); /* model string*/
	1276	ata_swap_string(port->identify + 23, 8); /* firmware string*/
	1277	ata_swap_string(port->identify + 10, 20); /* serial# string*/
	1278	#else
	1279	{
	1280	int i;
	1281	for (i = 0; i < ATA_ID_WORDS; i++)
	1282	port->identify[i] = le16_to_cpu(port->identify[i]);
	1283	}
	1284	#endif
	1285
	1286	/* Set the identify buffer as valid. */
	1287	port->identify_valid = 1;
	1288
	1289	if (user_buffer) {
	1290	if (copy_to_user(
	1291	user_buffer,
	1292	port->identify,
	1293	ATA_ID_WORDS * sizeof(u16))) {
	1294	rv = -EFAULT;
	1295	goto out;
	1296	}
	1297	}
	1298
	1299	out:
	1300	return rv;
	1301	}
	1302
	1303	/*
	1304	* Issue a standby immediate command to the device.
	1305	*
	1306	* @port Pointer to the port structure.
	1307	*
	1308	* return value
	1309	* 0 Command was executed successfully.
	1310	* -1 An error occurred while executing the command.
	1311	*/
	1312	static int mtip_standby_immediate(struct mtip_port *port)
	1313	{
	1314	int rv;
	1315	struct host_to_dev_fis fis;
	1316
	1317	/* Build the FIS. */
	1318	memset(&fis, 0, sizeof(struct host_to_dev_fis));
	1319	fis.type = 0x27;
	1320	fis.opts = 1 << 7;
	1321	fis.command = ATA_CMD_STANDBYNOW1;
	1322
	1323	/* Execute the command. Use a 15-second timeout for large drives. */
	1324	rv = mtip_exec_internal_command(port,
	1325	&fis,
	1326	5,
	1327	0,
	1328	0,
	1329	0,
	1330	GFP_KERNEL,
	1331	15000);
	1332
	1333	return rv;
	1334	}
	1335
	1336	/*
	1337	* Get the drive capacity.
	1338	*
	1339	* @dd Pointer to the device data structure.
	1340	* @sectors Pointer to the variable that will receive the sector count.
	1341	*
	1342	* return value
	1343	* 1 Capacity was returned successfully.
	1344	* 0 The identify information is invalid.
	1345	*/
	1346	static bool mtip_hw_get_capacity(struct driver_data dd, sector_t sectors)
	1347	{
	1348	struct mtip_port *port = dd->port;
	1349	u64 total, raw0, raw1, raw2, raw3;
	1350	raw0 = port->identify[100];
	1351	raw1 = port->identify[101];
	1352	raw2 = port->identify[102];
	1353	raw3 = port->identify[103];
	1354	total = raw0 \| raw1<<16 \| raw2<<32 \| raw3<<48;
	1355	*sectors = total;
	1356	return (bool) !!port->identify_valid;
	1357	}
	1358
	1359	/*
	1360	* Reset the HBA.
	1361	*
	1362	* Resets the HBA by setting the HBA Reset bit in the Global
	1363	* HBA Control register. After setting the HBA Reset bit the
	1364	* function waits for 1 second before reading the HBA Reset
	1365	* bit to make sure it has cleared. If HBA Reset is not clear
	1366	* an error is returned. Cannot be used in non-blockable
	1367	* context.
	1368	*
	1369	* @dd Pointer to the driver data structure.
	1370	*
	1371	* return value
	1372	* 0 The reset was successful.
	1373	* -1 The HBA Reset bit did not clear.
	1374	*/
	1375	static int mtip_hba_reset(struct driver_data *dd)
	1376	{
	1377	mtip_deinit_port(dd->port);
	1378
	1379	/* Set the reset bit */
	1380	writel(HOST_RESET, dd->mmio + HOST_CTL);
	1381
	1382	/* Flush */
	1383	readl(dd->mmio + HOST_CTL);
	1384
	1385	/* Wait for reset to clear */
	1386	ssleep(1);
	1387
	1388	/* Check the bit has cleared */
	1389	if (readl(dd->mmio + HOST_CTL) & HOST_RESET) {
	1390	dev_err(&dd->pdev->dev,
	1391	"Reset bit did not clear.\n");
	1392	return -1;
	1393	}
	1394
	1395	return 0;
	1396	}
	1397
	1398	/*
	1399	* Display the identify command data.
	1400	*
	1401	* @port Pointer to the port data structure.
	1402	*
	1403	* return value
	1404	* None
	1405	*/
	1406	static void mtip_dump_identify(struct mtip_port *port)
	1407	{
	1408	sector_t sectors;
	1409	unsigned short revid;
	1410	char cbuf[42];
	1411
	1412	if (!port->identify_valid)
	1413	return;
	1414
	1415	strlcpy(cbuf, (char *)(port->identify+10), 21);
	1416	dev_info(&port->dd->pdev->dev,
	1417	"Serial No.: %s\n", cbuf);
	1418
	1419	strlcpy(cbuf, (char *)(port->identify+23), 9);
	1420	dev_info(&port->dd->pdev->dev,
	1421	"Firmware Ver.: %s\n", cbuf);
	1422
	1423	strlcpy(cbuf, (char *)(port->identify+27), 41);
	1424	dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf);
	1425
	1426	if (mtip_hw_get_capacity(port->dd, &sectors))
	1427	dev_info(&port->dd->pdev->dev,
	1428	"Capacity: %llu sectors (%llu MB)\n",
	1429	(u64)sectors,
	1430	((u64)sectors) * ATA_SECT_SIZE >> 20);
	1431
	1432	pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid);
	1433	switch (revid & 0xFF) {
	1434	case 0x1:
	1435	strlcpy(cbuf, "A0", 3);
	1436	break;
	1437	case 0x3:
	1438	strlcpy(cbuf, "A2", 3);
	1439	break;
	1440	default:
	1441	strlcpy(cbuf, "?", 2);
	1442	break;
	1443	}
	1444	dev_info(&port->dd->pdev->dev,
	1445	"Card Type: %s\n", cbuf);
	1446	}
	1447
	1448	/*
	1449	* Map the commands scatter list into the command table.
	1450	*
	1451	* @command Pointer to the command.
	1452	* @nents Number of scatter list entries.
	1453	*
	1454	* return value
	1455	* None
	1456	*/
	1457	static inline void fill_command_sg(struct driver_data *dd,
	1458	struct mtip_cmd *command,
	1459	int nents)
	1460	{
	1461	int n;
	1462	unsigned int dma_len;
	1463	struct mtip_cmd_sg *command_sg;
	1464	struct scatterlist *sg = command->sg;
	1465
	1466	command_sg = command->command + AHCI_CMD_TBL_HDR_SZ;
	1467
	1468	for (n = 0; n < nents; n++) {
	1469	dma_len = sg_dma_len(sg);
	1470	if (dma_len > 0x400000)
	1471	dev_err(&dd->pdev->dev,
	1472	"DMA segment length truncated\n");
	1473	command_sg->info = __force_bit2int
	1474	cpu_to_le32((dma_len-1) & 0x3FFFFF);
	1475	command_sg->dba = __force_bit2int
	1476	cpu_to_le32(sg_dma_address(sg));
	1477	command_sg->dba_upper = __force_bit2int
	1478	cpu_to_le32((sg_dma_address(sg) >> 16) >> 16);
	1479	command_sg++;
	1480	sg++;
	1481	}
	1482	}
	1483
	1484	/*
	1485	* @brief Execute a drive command.
	1486	*
	1487	* return value 0 The command completed successfully.
	1488	* return value -1 An error occurred while executing the command.
	1489	*/
	1490	static int exec_drive_task(struct mtip_port port, u8 command)
	1491	{
	1492	struct host_to_dev_fis fis;
	1493	struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG);
	1494
	1495	/* Build the FIS. */
	1496	memset(&fis, 0, sizeof(struct host_to_dev_fis));
	1497	fis.type = 0x27;
	1498	fis.opts = 1 << 7;
	1499	fis.command = command[0];
	1500	fis.features = command[1];
	1501	fis.sect_count = command[2];
	1502	fis.sector = command[3];
	1503	fis.cyl_low = command[4];
	1504	fis.cyl_hi = command[5];
	1505	fis.device = command[6] & ~0x10; /* Clear the dev bit*/
	1506
	1507
	1508	dbg_printk(MTIP_DRV_NAME "%s: User Command: cmd %x, feat %x, "
	1509	"nsect %x, sect %x, lcyl %x, "
	1510	"hcyl %x, sel %x\n",
	1511	__func__,
	1512	command[0],
	1513	command[1],
	1514	command[2],
	1515	command[3],
	1516	command[4],
	1517	command[5],
	1518	command[6]);
	1519
	1520	/* Execute the command. */
	1521	if (mtip_exec_internal_command(port,
	1522	&fis,
	1523	5,
	1524	0,
	1525	0,
	1526	0,
	1527	GFP_KERNEL,
	1528	MTIP_IOCTL_COMMAND_TIMEOUT_MS) < 0) {
	1529	return -1;
	1530	}
	1531
	1532	command[0] = reply->command; /* Status*/
	1533	command[1] = reply->features; /* Error*/
	1534	command[4] = reply->cyl_low;
	1535	command[5] = reply->cyl_hi;
	1536
	1537	dbg_printk(MTIP_DRV_NAME "%s: Completion Status: stat %x, "
	1538	"err %x , cyl_lo %x cyl_hi %x\n",
	1539	__func__,
	1540	command[0],
	1541	command[1],
	1542	command[4],
	1543	command[5]);
	1544
	1545	return 0;
	1546	}
	1547
	1548	/*
	1549	* @brief Execute a drive command.
	1550	*
	1551	* @param port Pointer to the port data structure.
	1552	* @param command Pointer to the user specified command parameters.
	1553	* @param user_buffer Pointer to the user space buffer where read sector
	1554	* data should be copied.
	1555	*
	1556	* return value 0 The command completed successfully.
	1557	* return value -EFAULT An error occurred while copying the completion
	1558	* data to the user space buffer.
	1559	* return value -1 An error occurred while executing the command.
	1560	*/
	1561	static int exec_drive_command(struct mtip_port port, u8 command,
	1562	void __user *user_buffer)
	1563	{
	1564	struct host_to_dev_fis fis;
	1565	struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG);
	1566
	1567	/* Build the FIS. */
	1568	memset(&fis, 0, sizeof(struct host_to_dev_fis));
	1569	fis.type = 0x27;
	1570	fis.opts = 1 << 7;
	1571	fis.command = command[0];
	1572	fis.features = command[2];
	1573	fis.sect_count = command[3];
	1574	if (fis.command == ATA_CMD_SMART) {
	1575	fis.sector = command[1];
	1576	fis.cyl_low = 0x4F;
	1577	fis.cyl_hi = 0xC2;
	1578	}
	1579
	1580	dbg_printk(MTIP_DRV_NAME
	1581	"%s: User Command: cmd %x, sect %x, "
	1582	"feat %x, sectcnt %x\n",
	1583	__func__,
	1584	command[0],
	1585	command[1],
	1586	command[2],
	1587	command[3]);
	1588
	1589	memset(port->sector_buffer, 0x00, ATA_SECT_SIZE);
	1590
	1591	/* Execute the command. */
	1592	if (mtip_exec_internal_command(port,
	1593	&fis,
	1594	5,
	1595	port->sector_buffer_dma,
	1596	(command[3] != 0) ? ATA_SECT_SIZE : 0,
	1597	0,
	1598	GFP_KERNEL,
	1599	MTIP_IOCTL_COMMAND_TIMEOUT_MS)
	1600	< 0) {
	1601	return -1;
	1602	}
	1603
	1604	/* Collect the completion status. */
	1605	command[0] = reply->command; /* Status*/
	1606	command[1] = reply->features; /* Error*/
	1607	command[2] = command[3];
	1608
	1609	dbg_printk(MTIP_DRV_NAME
	1610	"%s: Completion Status: stat %x, "
	1611	"err %x, cmd %x\n",
	1612	__func__,
	1613	command[0],
	1614	command[1],
	1615	command[2]);
	1616
	1617	if (user_buffer && command[3]) {
	1618	if (copy_to_user(user_buffer,
	1619	port->sector_buffer,
	1620	ATA_SECT_SIZE * command[3])) {
	1621	return -EFAULT;
	1622	}
	1623	}
	1624
	1625	return 0;
	1626	}
	1627
	1628	/*
	1629	* Indicates whether a command has a single sector payload.
	1630	*
	1631	* @command passed to the device to perform the certain event.
	1632	* @features passed to the device to perform the certain event.
	1633	*
	1634	* return value
	1635	* 1 command is one that always has a single sector payload,
	1636	* regardless of the value in the Sector Count field.
	1637	* 0 otherwise
	1638	*
	1639	*/
	1640	static unsigned int implicit_sector(unsigned char command,
	1641	unsigned char features)
	1642	{
	1643	unsigned int rv = 0;
	1644
	1645	/* list of commands that have an implicit sector count of 1 */
	1646	switch (command) {
	1647	case ATA_CMD_SEC_SET_PASS:
	1648	case ATA_CMD_SEC_UNLOCK:
	1649	case ATA_CMD_SEC_ERASE_PREP:
	1650	case ATA_CMD_SEC_ERASE_UNIT:
	1651	case ATA_CMD_SEC_FREEZE_LOCK:
	1652	case ATA_CMD_SEC_DISABLE_PASS:
	1653	case ATA_CMD_PMP_READ:
	1654	case ATA_CMD_PMP_WRITE:
	1655	rv = 1;
	1656	break;
	1657	case ATA_CMD_SET_MAX:
	1658	if (features == ATA_SET_MAX_UNLOCK)
	1659	rv = 1;
	1660	break;
	1661	case ATA_CMD_SMART:
	1662	if ((features == ATA_SMART_READ_VALUES) \|\|
	1663	(features == ATA_SMART_READ_THRESHOLDS))
	1664	rv = 1;
	1665	break;
	1666	case ATA_CMD_CONF_OVERLAY:
	1667	if ((features == ATA_DCO_IDENTIFY) \|\|
	1668	(features == ATA_DCO_SET))
	1669	rv = 1;
	1670	break;
	1671	}
	1672	return rv;
	1673	}
	1674
	1675	/*
	1676	* Executes a taskfile
	1677	* See ide_taskfile_ioctl() for derivation
	1678	*/
	1679	static int exec_drive_taskfile(struct driver_data *dd,
	1680	void __user *buf,
	1681	ide_task_request_t *req_task,
	1682	int outtotal)
	1683	{
	1684	struct host_to_dev_fis fis;
	1685	struct host_to_dev_fis *reply;
	1686	u8 *outbuf = NULL;
	1687	u8 *inbuf = NULL;
	1688	dma_addr_t outbuf_dma = 0;
	1689	dma_addr_t inbuf_dma = 0;
	1690	dma_addr_t dma_buffer = 0;
	1691	int err = 0;
	1692	unsigned int taskin = 0;
	1693	unsigned int taskout = 0;
	1694	u8 nsect = 0;
	1695	unsigned int timeout = MTIP_IOCTL_COMMAND_TIMEOUT_MS;
	1696	unsigned int force_single_sector;
	1697	unsigned int transfer_size;
	1698	unsigned long task_file_data;
	1699	int intotal = outtotal + req_task->out_size;
	1700
	1701	taskout = req_task->out_size;
	1702	taskin = req_task->in_size;
	1703	/* 130560 = 512 * 0xFF*/
	1704	if (taskin > 130560 \|\| taskout > 130560) {
	1705	err = -EINVAL;
	1706	goto abort;
	1707	}
	1708
	1709	if (taskout) {
	1710	outbuf = kzalloc(taskout, GFP_KERNEL);
	1711	if (outbuf == NULL) {
	1712	err = -ENOMEM;
	1713	goto abort;
	1714	}
	1715	if (copy_from_user(outbuf, buf + outtotal, taskout)) {
	1716	err = -EFAULT;
	1717	goto abort;
	1718	}
	1719	outbuf_dma = pci_map_single(dd->pdev,
	1720	outbuf,
	1721	taskout,
	1722	DMA_TO_DEVICE);
	1723	if (outbuf_dma == 0) {
	1724	err = -ENOMEM;
	1725	goto abort;
	1726	}
	1727	dma_buffer = outbuf_dma;
	1728	}
	1729
	1730	if (taskin) {
	1731	inbuf = kzalloc(taskin, GFP_KERNEL);
	1732	if (inbuf == NULL) {
	1733	err = -ENOMEM;
	1734	goto abort;
	1735	}
	1736
	1737	if (copy_from_user(inbuf, buf + intotal, taskin)) {
	1738	err = -EFAULT;
	1739	goto abort;
	1740	}
	1741	inbuf_dma = pci_map_single(dd->pdev,
	1742	inbuf,
	1743	taskin, DMA_FROM_DEVICE);
	1744	if (inbuf_dma == 0) {
	1745	err = -ENOMEM;
	1746	goto abort;
	1747	}
	1748	dma_buffer = inbuf_dma;
	1749	}
	1750
	1751	/* only supports PIO and non-data commands from this ioctl. */
	1752	switch (req_task->data_phase) {
	1753	case TASKFILE_OUT:
	1754	nsect = taskout / ATA_SECT_SIZE;
	1755	reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
	1756	break;
	1757	case TASKFILE_IN:
	1758	reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
	1759	break;
	1760	case TASKFILE_NO_DATA:
	1761	reply = (dd->port->rxfis + RX_FIS_D2H_REG);
	1762	break;
	1763	default:
	1764	err = -EINVAL;
	1765	goto abort;
	1766	}
	1767
	1768	/* Build the FIS. */
	1769	memset(&fis, 0, sizeof(struct host_to_dev_fis));
	1770
	1771	fis.type = 0x27;
	1772	fis.opts = 1 << 7;
	1773	fis.command = req_task->io_ports[7];
	1774	fis.features = req_task->io_ports[1];
	1775	fis.sect_count = req_task->io_ports[2];
	1776	fis.lba_low = req_task->io_ports[3];
	1777	fis.lba_mid = req_task->io_ports[4];
	1778	fis.lba_hi = req_task->io_ports[5];
	1779	/* Clear the dev bit*/
	1780	fis.device = req_task->io_ports[6] & ~0x10;
	1781
	1782	if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) {
	1783	req_task->in_flags.all =
	1784	IDE_TASKFILE_STD_IN_FLAGS \|
	1785	(IDE_HOB_STD_IN_FLAGS << 8);
	1786	fis.lba_low_ex = req_task->hob_ports[3];
	1787	fis.lba_mid_ex = req_task->hob_ports[4];
	1788	fis.lba_hi_ex = req_task->hob_ports[5];
	1789	fis.features_ex = req_task->hob_ports[1];
	1790	fis.sect_cnt_ex = req_task->hob_ports[2];
	1791
	1792	} else {
	1793	req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS;
	1794	}
	1795
	1796	force_single_sector = implicit_sector(fis.command, fis.features);
	1797
	1798	if ((taskin \|\| taskout) && (!fis.sect_count)) {
	1799	if (nsect)
	1800	fis.sect_count = nsect;
	1801	else {
	1802	if (!force_single_sector) {
	1803	dev_warn(&dd->pdev->dev,
	1804	"data movement but "
	1805	"sect_count is 0\n");
	1806	err = -EINVAL;
	1807	goto abort;
	1808	}
	1809	}
	1810	}
	1811
	1812	dbg_printk(MTIP_DRV_NAME
	1813	"taskfile: cmd %x, feat %x, nsect %x,"
	1814	" sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x,"
	1815	" head/dev %x\n",
	1816	fis.command,
	1817	fis.features,
	1818	fis.sect_count,
	1819	fis.lba_low,
	1820	fis.lba_mid,
	1821	fis.lba_hi,
	1822	fis.device);
	1823
	1824	switch (fis.command) {
	1825	case ATA_CMD_DOWNLOAD_MICRO:
	1826	/* Change timeout for Download Microcode to 60 seconds.*/
	1827	timeout = 60000;
	1828	break;
	1829	case ATA_CMD_SEC_ERASE_UNIT:
	1830	/* Change timeout for Security Erase Unit to 4 minutes.*/
	1831	timeout = 240000;
	1832	break;
	1833	case ATA_CMD_STANDBYNOW1:
	1834	/* Change timeout for standby immediate to 10 seconds.*/
	1835	timeout = 10000;
	1836	break;
	1837	case 0xF7:
	1838	case 0xFA:
	1839	/* Change timeout for vendor unique command to 10 secs */
	1840	timeout = 10000;
	1841	break;
	1842	case ATA_CMD_SMART:
	1843	/* Change timeout for vendor unique command to 10 secs */
	1844	timeout = 10000;
	1845	break;
	1846	default:
	1847	timeout = MTIP_IOCTL_COMMAND_TIMEOUT_MS;
	1848	break;
	1849	}
	1850
	1851	/* Determine the correct transfer size.*/
	1852	if (force_single_sector)
	1853	transfer_size = ATA_SECT_SIZE;
	1854	else
	1855	transfer_size = ATA_SECT_SIZE * fis.sect_count;
	1856
	1857	/* Execute the command.*/
	1858	if (mtip_exec_internal_command(dd->port,
	1859	&fis,
	1860	5,
	1861	dma_buffer,
	1862	transfer_size,
	1863	0,
	1864	GFP_KERNEL,
	1865	timeout) < 0) {
	1866	err = -EIO;
	1867	goto abort;
	1868	}
	1869
	1870	task_file_data = readl(dd->port->mmio+PORT_TFDATA);
	1871
	1872	if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) {
	1873	reply = dd->port->rxfis + RX_FIS_PIO_SETUP;
	1874	req_task->io_ports[7] = reply->control;
	1875	} else {
	1876	reply = dd->port->rxfis + RX_FIS_D2H_REG;
	1877	req_task->io_ports[7] = reply->command;
	1878	}
	1879
	1880	/* reclaim the DMA buffers.*/
	1881	if (inbuf_dma)
	1882	pci_unmap_single(dd->pdev, inbuf_dma,
	1883	taskin, DMA_FROM_DEVICE);
	1884	if (outbuf_dma)
	1885	pci_unmap_single(dd->pdev, outbuf_dma,
	1886	taskout, DMA_TO_DEVICE);
	1887	inbuf_dma = 0;
	1888	outbuf_dma = 0;
	1889
	1890	/* return the ATA registers to the caller.*/
	1891	req_task->io_ports[1] = reply->features;
	1892	req_task->io_ports[2] = reply->sect_count;
	1893	req_task->io_ports[3] = reply->lba_low;
	1894	req_task->io_ports[4] = reply->lba_mid;
	1895	req_task->io_ports[5] = reply->lba_hi;
	1896	req_task->io_ports[6] = reply->device;
	1897
	1898	if (req_task->out_flags.all & 1) {
	1899
	1900	req_task->hob_ports[3] = reply->lba_low_ex;
	1901	req_task->hob_ports[4] = reply->lba_mid_ex;
	1902	req_task->hob_ports[5] = reply->lba_hi_ex;
	1903	req_task->hob_ports[1] = reply->features_ex;
	1904	req_task->hob_ports[2] = reply->sect_cnt_ex;
	1905	}
	1906
	1907	/* Com rest after secure erase or lowlevel format */
	1908	if (((fis.command == ATA_CMD_SEC_ERASE_UNIT) \|\|
	1909	((fis.command == 0xFC) &&
	1910	(fis.features == 0x27 \|\| fis.features == 0x72 \|\|
	1911	fis.features == 0x62 \|\| fis.features == 0x26))) &&
	1912	!(reply->command & 1)) {
	1913	mtip_restart_port(dd->port);
	1914	}
	1915
	1916	dbg_printk(MTIP_DRV_NAME
	1917	"%s: Completion: stat %x,"
	1918	"err %x, sect_cnt %x, lbalo %x,"
	1919	"lbamid %x, lbahi %x, dev %x\n",
	1920	__func__,
	1921	req_task->io_ports[7],
	1922	req_task->io_ports[1],
	1923	req_task->io_ports[2],
	1924	req_task->io_ports[3],
	1925	req_task->io_ports[4],
	1926	req_task->io_ports[5],
	1927	req_task->io_ports[6]);
	1928
	1929	if (taskout) {
	1930	if (copy_to_user(buf + outtotal, outbuf, taskout)) {
	1931	err = -EFAULT;
	1932	goto abort;
	1933	}
	1934	}
	1935	if (taskin) {
	1936	if (copy_to_user(buf + intotal, inbuf, taskin)) {
	1937	err = -EFAULT;
	1938	goto abort;
	1939	}
	1940	}
	1941	abort:
	1942	if (inbuf_dma)
	1943	pci_unmap_single(dd->pdev, inbuf_dma,
	1944	taskin, DMA_FROM_DEVICE);
	1945	if (outbuf_dma)
	1946	pci_unmap_single(dd->pdev, outbuf_dma,
	1947	taskout, DMA_TO_DEVICE);
	1948	kfree(outbuf);
	1949	kfree(inbuf);
	1950
	1951	return err;
	1952	}
	1953
	1954	/*
	1955	* Handle IOCTL calls from the Block Layer.
	1956	*
	1957	* This function is called by the Block Layer when it receives an IOCTL
	1958	* command that it does not understand. If the IOCTL command is not supported
	1959	* this function returns -ENOTTY.
	1960	*
	1961	* @dd Pointer to the driver data structure.
	1962	* @cmd IOCTL command passed from the Block Layer.
	1963	* @arg IOCTL argument passed from the Block Layer.
	1964	*
	1965	* return value
	1966	* 0 The IOCTL completed successfully.
	1967	* -ENOTTY The specified command is not supported.
	1968	* -EFAULT An error occurred copying data to a user space buffer.
	1969	* -EIO An error occurred while executing the command.
	1970	*/
	1971	static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd,
	1972	unsigned long arg)
	1973	{
	1974	switch (cmd) {
	1975	case HDIO_GET_IDENTITY:
	1976	if (mtip_get_identify(dd->port, (void __user *) arg) < 0) {
	1977	dev_warn(&dd->pdev->dev,
	1978	"Unable to read identity\n");
	1979	return -EIO;
	1980	}
	1981
	1982	break;
	1983	case HDIO_DRIVE_CMD:
	1984	{
	1985	u8 drive_command[4];
	1986
	1987	/* Copy the user command info to our buffer. */
	1988	if (copy_from_user(drive_command,
	1989	(void __user *) arg,
	1990	sizeof(drive_command)))
	1991	return -EFAULT;
	1992
	1993	/* Execute the drive command. */
	1994	if (exec_drive_command(dd->port,
	1995	drive_command,
	1996	(void __user *) (arg+4)))
	1997	return -EIO;
	1998
	1999	/* Copy the status back to the users buffer. */
	2000	if (copy_to_user((void __user *) arg,
	2001	drive_command,
	2002	sizeof(drive_command)))
	2003	return -EFAULT;
	2004
	2005	break;
	2006	}
	2007	case HDIO_DRIVE_TASK:
	2008	{
	2009	u8 drive_command[7];
	2010
	2011	/* Copy the user command info to our buffer. */
	2012	if (copy_from_user(drive_command,
	2013	(void __user *) arg,
	2014	sizeof(drive_command)))
	2015	return -EFAULT;
	2016
	2017	/* Execute the drive command. */
	2018	if (exec_drive_task(dd->port, drive_command))
	2019	return -EIO;
	2020
	2021	/* Copy the status back to the users buffer. */
	2022	if (copy_to_user((void __user *) arg,
	2023	drive_command,
	2024	sizeof(drive_command)))
	2025	return -EFAULT;
	2026
	2027	break;
	2028	}
	2029	case HDIO_DRIVE_TASKFILE: {
	2030	ide_task_request_t req_task;
	2031	int ret, outtotal;
	2032
	2033	if (copy_from_user(&req_task, (void __user *) arg,
	2034	sizeof(req_task)))
	2035	return -EFAULT;
	2036
	2037	outtotal = sizeof(req_task);
	2038
	2039	ret = exec_drive_taskfile(dd, (void __user *) arg,
	2040	&req_task, outtotal);
	2041
	2042	if (copy_to_user((void __user *) arg, &req_task,
	2043	sizeof(req_task)))
	2044	return -EFAULT;
	2045
	2046	return ret;
	2047	}
	2048
	2049	default:
	2050	return -EINVAL;
	2051	}
	2052	return 0;
	2053	}
	2054
	2055	/*
	2056	* Submit an IO to the hw
	2057	*
	2058	* This function is called by the block layer to issue an io
	2059	* to the device. Upon completion, the callback function will
	2060	* be called with the data parameter passed as the callback data.
	2061	*
	2062	* @dd Pointer to the driver data structure.
	2063	* @start First sector to read.
	2064	* @nsect Number of sectors to read.
	2065	* @nents Number of entries in scatter list for the read command.
	2066	* @tag The tag of this read command.
	2067	* @callback Pointer to the function that should be called
	2068	* when the read completes.
	2069	* @data Callback data passed to the callback function
	2070	* when the read completes.
	2071	* @barrier If non-zero, this command must be completed before
	2072	* issuing any other commands.
	2073	* @dir Direction (read or write)
	2074	*
	2075	* return value
	2076	* None
	2077	*/
	2078	static void mtip_hw_submit_io(struct driver_data *dd, sector_t start,
	2079	int nsect, int nents, int tag, void *callback,
	2080	void *data, int barrier, int dir)
	2081	{
	2082	struct host_to_dev_fis *fis;
	2083	struct mtip_port *port = dd->port;
	2084	struct mtip_cmd *command = &port->commands[tag];
	2085
	2086	/* Map the scatter list for DMA access */
	2087	if (dir == READ)
	2088	nents = dma_map_sg(&dd->pdev->dev, command->sg,
	2089	nents, DMA_FROM_DEVICE);
	2090	else
	2091	nents = dma_map_sg(&dd->pdev->dev, command->sg,
	2092	nents, DMA_TO_DEVICE);
	2093
	2094	command->scatter_ents = nents;
	2095
	2096	/*
	2097	* The number of retries for this command before it is
	2098	* reported as a failure to the upper layers.
	2099	*/
	2100	command->retries = MTIP_MAX_RETRIES;
	2101
	2102	/* Fill out fis */
	2103	fis = command->command;
	2104	fis->type = 0x27;
	2105	fis->opts = 1 << 7;
	2106	fis->command =
	2107	(dir == READ ? ATA_CMD_FPDMA_READ : ATA_CMD_FPDMA_WRITE);
	2108	((unsigned int ) &fis->lba_low) = (start & 0xFFFFFF);
	2109	((unsigned int ) &fis->lba_low_ex) = ((start >> 24) & 0xFFFFFF);
	2110	fis->device = 1 << 6;
	2111	if (barrier)
	2112	fis->device \|= FUA_BIT;
	2113	fis->features = nsect & 0xFF;
	2114	fis->features_ex = (nsect >> 8) & 0xFF;
	2115	fis->sect_count = ((tag << 3) \| (tag >> 5));
	2116	fis->sect_cnt_ex = 0;
	2117	fis->control = 0;
	2118	fis->res2 = 0;
	2119	fis->res3 = 0;
	2120	fill_command_sg(dd, command, nents);
	2121
	2122	/* Populate the command header */
	2123	command->command_header->opts =
	2124	__force_bit2int cpu_to_le32(
	2125	(nents << 16) \| 5 \| AHCI_CMD_PREFETCH);
	2126	command->command_header->byte_count = 0;
	2127
	2128	/*
	2129	* Set the completion function and data for the command
	2130	* within this layer.
	2131	*/
	2132	command->comp_data = dd;
	2133	command->comp_func = mtip_async_complete;
	2134	command->direction = (dir == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
	2135
	2136	/*
	2137	* Set the completion function and data for the command passed
	2138	* from the upper layer.
	2139	*/
	2140	command->async_data = data;
	2141	command->async_callback = callback;
	2142
	2143	/*
	2144	* To prevent this command from being issued
	2145	* if an internal command is in progress or error handling is active.
	2146	*/
	2147	if (unlikely(test_bit(MTIP_FLAG_IC_ACTIVE_BIT, &port->flags) \|\|
	2148	test_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags))) {
	2149	set_bit(tag, port->cmds_to_issue);
	2150	set_bit(MTIP_FLAG_ISSUE_CMDS_BIT, &port->flags);
	2151	return;
	2152	}
	2153
	2154	/* Issue the command to the hardware */
	2155	mtip_issue_ncq_command(port, tag);
	2156
	2157	/* Set the command's timeout value.*/
	2158	port->commands[tag].comp_time = jiffies + msecs_to_jiffies(
	2159	MTIP_NCQ_COMMAND_TIMEOUT_MS);
	2160	}
	2161
	2162	/*
	2163	* Release a command slot.
	2164	*
	2165	* @dd Pointer to the driver data structure.
	2166	* @tag Slot tag
	2167	*
	2168	* return value
	2169	* None
	2170	*/
	2171	static void mtip_hw_release_scatterlist(struct driver_data *dd, int tag)
	2172	{
	2173	release_slot(dd->port, tag);
	2174	}
	2175
	2176	/*
	2177	* Obtain a command slot and return its associated scatter list.
	2178	*
	2179	* @dd Pointer to the driver data structure.
	2180	* @tag Pointer to an int that will receive the allocated command
	2181	* slot tag.
	2182	*
	2183	* return value
	2184	* Pointer to the scatter list for the allocated command slot
	2185	* or NULL if no command slots are available.
	2186	*/
	2187	static struct scatterlist mtip_hw_get_scatterlist(struct driver_data dd,
	2188	int *tag)
	2189	{
	2190	/*
	2191	* It is possible that, even with this semaphore, a thread
	2192	* may think that no command slots are available. Therefore, we
	2193	* need to make an attempt to get_slot().
	2194	*/
	2195	down(&dd->port->cmd_slot);
	2196	*tag = get_slot(dd->port);
	2197
	2198	if (unlikely(*tag < 0))
	2199	return NULL;
	2200
	2201	return dd->port->commands[*tag].sg;
	2202	}
	2203
	2204	/*
	2205	* Sysfs register/status dump.
	2206	*
	2207	* @dev Pointer to the device structure, passed by the kernrel.
	2208	* @attr Pointer to the device_attribute structure passed by the kernel.
	2209	* @buf Pointer to the char buffer that will receive the stats info.
	2210	*
	2211	* return value
	2212	* The size, in bytes, of the data copied into buf.
	2213	*/
	2214	static ssize_t hw_show_registers(struct device *dev,
	2215	struct device_attribute *attr,
	2216	char *buf)
	2217	{
	2218	u32 group_allocated;
	2219	struct driver_data *dd = dev_to_disk(dev)->private_data;
	2220	int size = 0;
	2221	int n;
	2222
	2223	size += sprintf(&buf[size], "%s:\ns_active:\n", __func__);
	2224
	2225	for (n = 0; n < dd->slot_groups; n++)
	2226	size += sprintf(&buf[size], "0x%08x\n",
	2227	readl(dd->port->s_active[n]));
	2228
	2229	size += sprintf(&buf[size], "Command Issue:\n");
	2230
	2231	for (n = 0; n < dd->slot_groups; n++)
	2232	size += sprintf(&buf[size], "0x%08x\n",
	2233	readl(dd->port->cmd_issue[n]));
	2234
	2235	size += sprintf(&buf[size], "Allocated:\n");
	2236
	2237	for (n = 0; n < dd->slot_groups; n++) {
	2238	if (sizeof(long) > sizeof(u32))
	2239	group_allocated =
	2240	dd->port->allocated[n/2] >> (32*(n&1));
	2241	else
	2242	group_allocated = dd->port->allocated[n];
	2243	size += sprintf(&buf[size], "0x%08x\n",
	2244	group_allocated);
	2245	}
	2246
	2247	size += sprintf(&buf[size], "completed:\n");
	2248
	2249	for (n = 0; n < dd->slot_groups; n++)
	2250	size += sprintf(&buf[size], "0x%08x\n",
	2251	readl(dd->port->completed[n]));
	2252
	2253	size += sprintf(&buf[size], "PORT_IRQ_STAT 0x%08x\n",
	2254	readl(dd->port->mmio + PORT_IRQ_STAT));
	2255	size += sprintf(&buf[size], "HOST_IRQ_STAT 0x%08x\n",
	2256	readl(dd->mmio + HOST_IRQ_STAT));
	2257
	2258	return size;
	2259	}
	2260	static DEVICE_ATTR(registers, S_IRUGO, hw_show_registers, NULL);
	2261
	2262	/*
	2263	* Create the sysfs related attributes.
	2264	*
	2265	* @dd Pointer to the driver data structure.
	2266	* @kobj Pointer to the kobj for the block device.
	2267	*
	2268	* return value
	2269	* 0 Operation completed successfully.
	2270	* -EINVAL Invalid parameter.
	2271	*/
	2272	static int mtip_hw_sysfs_init(struct driver_data dd, struct kobject kobj)
	2273	{
	2274	if (!kobj \|\| !dd)
	2275	return -EINVAL;
	2276
	2277	if (sysfs_create_file(kobj, &dev_attr_registers.attr))
	2278	dev_warn(&dd->pdev->dev,
	2279	"Error creating registers sysfs entry\n");
	2280	return 0;
	2281	}
	2282
	2283	/*
	2284	* Remove the sysfs related attributes.
	2285	*
	2286	* @dd Pointer to the driver data structure.
	2287	* @kobj Pointer to the kobj for the block device.
	2288	*
	2289	* return value
	2290	* 0 Operation completed successfully.
	2291	* -EINVAL Invalid parameter.
	2292	*/
	2293	static int mtip_hw_sysfs_exit(struct driver_data dd, struct kobject kobj)
	2294	{
	2295	if (!kobj \|\| !dd)
	2296	return -EINVAL;
	2297
	2298	sysfs_remove_file(kobj, &dev_attr_registers.attr);
	2299
	2300	return 0;
	2301	}
	2302
	2303	/*
	2304	* Perform any init/resume time hardware setup
	2305	*
	2306	* @dd Pointer to the driver data structure.
	2307	*
	2308	* return value
	2309	* None
	2310	*/
	2311	static inline void hba_setup(struct driver_data *dd)
	2312	{
	2313	u32 hwdata;
	2314	hwdata = readl(dd->mmio + HOST_HSORG);
	2315
	2316	/* interrupt bug workaround: use only 1 IS bit.*/
	2317	writel(hwdata \|
	2318	HSORG_DISABLE_SLOTGRP_INTR \|
	2319	HSORG_DISABLE_SLOTGRP_PXIS,
	2320	dd->mmio + HOST_HSORG);
	2321	}
	2322
	2323	/*
	2324	* Detect the details of the product, and store anything needed
	2325	* into the driver data structure. This includes product type and
	2326	* version and number of slot groups.
	2327	*
	2328	* @dd Pointer to the driver data structure.
	2329	*
	2330	* return value
	2331	* None
	2332	*/
	2333	static void mtip_detect_product(struct driver_data *dd)
	2334	{
	2335	u32 hwdata;
	2336	unsigned int rev, slotgroups;
	2337
	2338	/*
	2339	* HBA base + 0xFC [15:0] - vendor-specific hardware interface
	2340	* info register:
	2341	* [15:8] hardware/software interface rev#
	2342	* [ 3] asic-style interface
	2343	* [ 2:0] number of slot groups, minus 1 (only valid for asic-style).
	2344	*/
	2345	hwdata = readl(dd->mmio + HOST_HSORG);
	2346
	2347	dd->product_type = MTIP_PRODUCT_UNKNOWN;
	2348	dd->slot_groups = 1;
	2349
	2350	if (hwdata & 0x8) {
	2351	dd->product_type = MTIP_PRODUCT_ASICFPGA;
	2352	rev = (hwdata & HSORG_HWREV) >> 8;
	2353	slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1;
	2354	dev_info(&dd->pdev->dev,
	2355	"ASIC-FPGA design, HS rev 0x%x, "
	2356	"%i slot groups [%i slots]\n",
	2357	rev,
	2358	slotgroups,
	2359	slotgroups * 32);
	2360
	2361	if (slotgroups > MTIP_MAX_SLOT_GROUPS) {
	2362	dev_warn(&dd->pdev->dev,
	2363	"Warning: driver only supports "
	2364	"%i slot groups.\n", MTIP_MAX_SLOT_GROUPS);
	2365	slotgroups = MTIP_MAX_SLOT_GROUPS;
	2366	}
	2367	dd->slot_groups = slotgroups;
	2368	return;
	2369	}
	2370
	2371	dev_warn(&dd->pdev->dev, "Unrecognized product id\n");
	2372	}
	2373
	2374	/*
	2375	* Blocking wait for FTL rebuild to complete
	2376	*
	2377	* @dd Pointer to the DRIVER_DATA structure.
	2378	*
	2379	* return value
	2380	* 0 FTL rebuild completed successfully
	2381	* -EFAULT FTL rebuild error/timeout/interruption
	2382	*/
	2383	static int mtip_ftl_rebuild_poll(struct driver_data *dd)
	2384	{
	2385	unsigned long timeout, cnt = 0, start;
	2386
	2387	dev_warn(&dd->pdev->dev,
	2388	"FTL rebuild in progress. Polling for completion.\n");
	2389
	2390	start = jiffies;
	2391	dd->ftlrebuildflag = 1;
	2392	timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS);
	2393
	2394	do {
	2395	if (mtip_check_surprise_removal(dd->pdev))
	2396	return -EFAULT;
	2397
	2398	if (mtip_get_identify(dd->port, NULL) < 0)
	2399	return -EFAULT;
	2400
	2401	if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
	2402	MTIP_FTL_REBUILD_MAGIC) {
	2403	ssleep(1);
	2404	/* Print message every 3 minutes */
	2405	if (cnt++ >= 180) {
	2406	dev_warn(&dd->pdev->dev,
	2407	"FTL rebuild in progress (%d secs).\n",
	2408	jiffies_to_msecs(jiffies - start) / 1000);
	2409	cnt = 0;
	2410	}
	2411	} else {
	2412	dev_warn(&dd->pdev->dev,
	2413	"FTL rebuild complete (%d secs).\n",
	2414	jiffies_to_msecs(jiffies - start) / 1000);
	2415	dd->ftlrebuildflag = 0;
	2416	mtip_block_initialize(dd);
	2417	break;
	2418	}
	2419	ssleep(10);
	2420	} while (time_before(jiffies, timeout));
	2421
	2422	/* Check for timeout */
	2423	if (dd->ftlrebuildflag) {
	2424	dev_err(&dd->pdev->dev,
	2425	"Timed out waiting for FTL rebuild to complete (%d secs).\n",
	2426	jiffies_to_msecs(jiffies - start) / 1000);
	2427	return -EFAULT;
	2428	}
	2429
	2430	return 0;
	2431	}
	2432
	2433	/*
	2434	* service thread to issue queued commands
	2435	*
	2436	* @data Pointer to the driver data structure.
	2437	*
	2438	* return value
	2439	* 0
	2440	*/
	2441
	2442	static int mtip_service_thread(void *data)
	2443	{
	2444	struct driver_data dd = (struct driver_data )data;
	2445	unsigned long slot, slot_start, slot_wrap;
	2446	unsigned int num_cmd_slots = dd->slot_groups * 32;
	2447	struct mtip_port *port = dd->port;
	2448
	2449	while (1) {
	2450	/*
	2451	* the condition is to check neither an internal command is
	2452	* is in progress nor error handling is active
	2453	*/
	2454	wait_event_interruptible(port->svc_wait, (port->flags) &&
	2455	!test_bit(MTIP_FLAG_IC_ACTIVE_BIT, &port->flags) &&
	2456	!test_bit(MTIP_FLAG_EH_ACTIVE_BIT, &port->flags));
	2457
	2458	if (kthread_should_stop())
	2459	break;
	2460
	2461	set_bit(MTIP_FLAG_SVC_THD_ACTIVE_BIT, &port->flags);
	2462	if (test_bit(MTIP_FLAG_ISSUE_CMDS_BIT, &port->flags)) {
	2463	slot = 1;
	2464	/* used to restrict the loop to one iteration */
	2465	slot_start = num_cmd_slots;
	2466	slot_wrap = 0;
	2467	while (1) {
	2468	slot = find_next_bit(port->cmds_to_issue,
	2469	num_cmd_slots, slot);
	2470	if (slot_wrap == 1) {
	2471	if ((slot_start >= slot) \|\|
	2472	(slot >= num_cmd_slots))
	2473	break;
	2474	}
	2475	if (unlikely(slot_start == num_cmd_slots))
	2476	slot_start = slot;
	2477
	2478	if (unlikely(slot == num_cmd_slots)) {
	2479	slot = 1;
	2480	slot_wrap = 1;
	2481	continue;
	2482	}
	2483
	2484	/* Issue the command to the hardware */
	2485	mtip_issue_ncq_command(port, slot);
	2486
	2487	/* Set the command's timeout value.*/
	2488	port->commands[slot].comp_time = jiffies +
	2489	msecs_to_jiffies(MTIP_NCQ_COMMAND_TIMEOUT_MS);
	2490
	2491	clear_bit(slot, port->cmds_to_issue);
	2492	}
	2493
	2494	clear_bit(MTIP_FLAG_ISSUE_CMDS_BIT, &port->flags);
	2495	} else if (test_bit(MTIP_FLAG_REBUILD_BIT, &port->flags)) {
	2496	mtip_ftl_rebuild_poll(dd);
	2497	clear_bit(MTIP_FLAG_REBUILD_BIT, &port->flags);
	2498	}
	2499	clear_bit(MTIP_FLAG_SVC_THD_ACTIVE_BIT, &port->flags);
	2500
	2501	if (test_bit(MTIP_FLAG_SVC_THD_SHOULD_STOP_BIT, &port->flags))
	2502	break;
	2503	}
	2504	return 0;
	2505	}
	2506
	2507	/*
	2508	* Called once for each card.
	2509	*
	2510	* @dd Pointer to the driver data structure.
	2511	*
	2512	* return value
	2513	* 0 on success, else an error code.
	2514	*/
	2515	static int mtip_hw_init(struct driver_data *dd)
	2516	{
	2517	int i;
	2518	int rv;
	2519	unsigned int num_command_slots;
	2520
	2521	dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR];
	2522
	2523	mtip_detect_product(dd);
	2524	if (dd->product_type == MTIP_PRODUCT_UNKNOWN) {
	2525	rv = -EIO;
	2526	goto out1;
	2527	}
	2528	num_command_slots = dd->slot_groups * 32;
	2529
	2530	hba_setup(dd);
	2531
	2532	tasklet_init(&dd->tasklet, mtip_tasklet, (unsigned long)dd);
	2533
	2534	dd->port = kzalloc(sizeof(struct mtip_port), GFP_KERNEL);
	2535	if (!dd->port) {
	2536	dev_err(&dd->pdev->dev,
	2537	"Memory allocation: port structure\n");
	2538	return -ENOMEM;
	2539	}
	2540
	2541	/* Counting semaphore to track command slot usage */
	2542	sema_init(&dd->port->cmd_slot, num_command_slots - 1);
	2543
	2544	/* Spinlock to prevent concurrent issue */
	2545	spin_lock_init(&dd->port->cmd_issue_lock);
	2546
	2547	/* Set the port mmio base address. */
	2548	dd->port->mmio = dd->mmio + PORT_OFFSET;
	2549	dd->port->dd = dd;
	2550
	2551	/* Allocate memory for the command list. */
	2552	dd->port->command_list =
	2553	dmam_alloc_coherent(&dd->pdev->dev,
	2554	HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 2),
	2555	&dd->port->command_list_dma,
	2556	GFP_KERNEL);
	2557	if (!dd->port->command_list) {
	2558	dev_err(&dd->pdev->dev,
	2559	"Memory allocation: command list\n");
	2560	rv = -ENOMEM;
	2561	goto out1;
	2562	}
	2563
	2564	/* Clear the memory we have allocated. */
	2565	memset(dd->port->command_list,
	2566	0,
	2567	HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 2));
	2568
	2569	/* Setup the addresse of the RX FIS. */
	2570	dd->port->rxfis = dd->port->command_list + HW_CMD_SLOT_SZ;
	2571	dd->port->rxfis_dma = dd->port->command_list_dma + HW_CMD_SLOT_SZ;
	2572
	2573	/* Setup the address of the command tables. */
	2574	dd->port->command_table = dd->port->rxfis + AHCI_RX_FIS_SZ;
	2575	dd->port->command_tbl_dma = dd->port->rxfis_dma + AHCI_RX_FIS_SZ;
	2576
	2577	/* Setup the address of the identify data. */
	2578	dd->port->identify = dd->port->command_table +
	2579	HW_CMD_TBL_AR_SZ;
	2580	dd->port->identify_dma = dd->port->command_tbl_dma +
	2581	HW_CMD_TBL_AR_SZ;
	2582
	2583	/* Setup the address of the sector buffer. */
	2584	dd->port->sector_buffer = (void *) dd->port->identify + ATA_SECT_SIZE;
	2585	dd->port->sector_buffer_dma = dd->port->identify_dma + ATA_SECT_SIZE;
	2586
	2587	/* Point the command headers at the command tables. */
	2588	for (i = 0; i < num_command_slots; i++) {
	2589	dd->port->commands[i].command_header =
	2590	dd->port->command_list +
	2591	(sizeof(struct mtip_cmd_hdr) * i);
	2592	dd->port->commands[i].command_header_dma =
	2593	dd->port->command_list_dma +
	2594	(sizeof(struct mtip_cmd_hdr) * i);
	2595
	2596	dd->port->commands[i].command =
	2597	dd->port->command_table + (HW_CMD_TBL_SZ * i);
	2598	dd->port->commands[i].command_dma =
	2599	dd->port->command_tbl_dma + (HW_CMD_TBL_SZ * i);
	2600
	2601	if (readl(dd->mmio + HOST_CAP) & HOST_CAP_64)
	2602	dd->port->commands[i].command_header->ctbau =
	2603	__force_bit2int cpu_to_le32(
	2604	(dd->port->commands[i].command_dma >> 16) >> 16);
	2605	dd->port->commands[i].command_header->ctba =
	2606	__force_bit2int cpu_to_le32(
	2607	dd->port->commands[i].command_dma & 0xFFFFFFFF);
	2608
	2609	/*
	2610	* If this is not done, a bug is reported by the stock
	2611	* FC11 i386. Due to the fact that it has lots of kernel
	2612	* debugging enabled.
	2613	*/
	2614	sg_init_table(dd->port->commands[i].sg, MTIP_MAX_SG);
	2615
	2616	/* Mark all commands as currently inactive.*/
	2617	atomic_set(&dd->port->commands[i].active, 0);
	2618	}
	2619
	2620	/* Setup the pointers to the extended s_active and CI registers. */
	2621	for (i = 0; i < dd->slot_groups; i++) {
	2622	dd->port->s_active[i] =
	2623	dd->port->mmio + i*0x80 + PORT_SCR_ACT;
	2624	dd->port->cmd_issue[i] =
	2625	dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE;
	2626	dd->port->completed[i] =
	2627	dd->port->mmio + i*0x80 + PORT_SDBV;
	2628	}
	2629
	2630	/* Reset the HBA. */
	2631	if (mtip_hba_reset(dd) < 0) {
	2632	dev_err(&dd->pdev->dev,
	2633	"Card did not reset within timeout\n");
	2634	rv = -EIO;
	2635	goto out2;
	2636	}
	2637
	2638	mtip_init_port(dd->port);
	2639	mtip_start_port(dd->port);
	2640
	2641	/* Setup the ISR and enable interrupts. */
	2642	rv = devm_request_irq(&dd->pdev->dev,
	2643	dd->pdev->irq,
	2644	mtip_irq_handler,
	2645	IRQF_SHARED,
	2646	dev_driver_string(&dd->pdev->dev),
	2647	dd);
	2648
	2649	if (rv) {
	2650	dev_err(&dd->pdev->dev,
	2651	"Unable to allocate IRQ %d\n", dd->pdev->irq);
	2652	goto out2;
	2653	}
	2654
	2655	/* Enable interrupts on the HBA. */
	2656	writel(readl(dd->mmio + HOST_CTL) \| HOST_IRQ_EN,
	2657	dd->mmio + HOST_CTL);
	2658
	2659	init_timer(&dd->port->cmd_timer);
	2660	init_waitqueue_head(&dd->port->svc_wait);
	2661
	2662	dd->port->cmd_timer.data = (unsigned long int) dd->port;
	2663	dd->port->cmd_timer.function = mtip_timeout_function;
	2664	mod_timer(&dd->port->cmd_timer,
	2665	jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD));
	2666
	2667	if (mtip_get_identify(dd->port, NULL) < 0) {
	2668	rv = -EFAULT;
	2669	goto out3;
	2670	}
	2671
	2672	if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
	2673	MTIP_FTL_REBUILD_MAGIC) {
	2674	set_bit(MTIP_FLAG_REBUILD_BIT, &dd->port->flags);
	2675	return MTIP_FTL_REBUILD_MAGIC;
	2676	}
	2677	mtip_dump_identify(dd->port);
	2678	return rv;
	2679
	2680	out3:
	2681	del_timer_sync(&dd->port->cmd_timer);
	2682
	2683	/* Disable interrupts on the HBA. */
	2684	writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
	2685	dd->mmio + HOST_CTL);
	2686
	2687	/Release the IRQ. /
	2688	devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd);
	2689
	2690	out2:
	2691	mtip_deinit_port(dd->port);
	2692
	2693	/* Free the command/command header memory. */
	2694	dmam_free_coherent(&dd->pdev->dev,
	2695	HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 2),
	2696	dd->port->command_list,
	2697	dd->port->command_list_dma);
	2698	out1:
	2699	/* Free the memory allocated for the for structure. */
	2700	kfree(dd->port);
	2701
	2702	return rv;
	2703	}
	2704
	2705	/*
	2706	* Called to deinitialize an interface.
	2707	*
	2708	* @dd Pointer to the driver data structure.
	2709	*
	2710	* return value
	2711	* 0
	2712	*/
	2713	static int mtip_hw_exit(struct driver_data *dd)
	2714	{
	2715	/*
	2716	* Send standby immediate (E0h) to the drive so that it
	2717	* saves its state.
	2718	*/
	2719	if (atomic_read(&dd->drv_cleanup_done) != true) {
	2720
	2721	mtip_standby_immediate(dd->port);
	2722
	2723	/* de-initialize the port. */
	2724	mtip_deinit_port(dd->port);
	2725
	2726	/* Disable interrupts on the HBA. */
	2727	writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
	2728	dd->mmio + HOST_CTL);
	2729	}
	2730
	2731	del_timer_sync(&dd->port->cmd_timer);
	2732
	2733	/* Release the IRQ. */
	2734	devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd);
	2735
	2736	/* Stop the bottom half tasklet. */
	2737	tasklet_kill(&dd->tasklet);
	2738
	2739	/* Free the command/command header memory. */
	2740	dmam_free_coherent(&dd->pdev->dev,
	2741	HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 2),
	2742	dd->port->command_list,
	2743	dd->port->command_list_dma);
	2744	/* Free the memory allocated for the for structure. */
	2745	kfree(dd->port);
	2746
	2747	return 0;
	2748	}
	2749
	2750	/*
	2751	* Issue a Standby Immediate command to the device.
	2752	*
	2753	* This function is called by the Block Layer just before the
	2754	* system powers off during a shutdown.
	2755	*
	2756	* @dd Pointer to the driver data structure.
	2757	*
	2758	* return value
	2759	* 0
	2760	*/
	2761	static int mtip_hw_shutdown(struct driver_data *dd)
	2762	{
	2763	/*
	2764	* Send standby immediate (E0h) to the drive so that it
	2765	* saves its state.
	2766	*/
	2767	mtip_standby_immediate(dd->port);
	2768
	2769	return 0;
	2770	}
	2771
	2772	/*
	2773	* Suspend function
	2774	*
	2775	* This function is called by the Block Layer just before the
	2776	* system hibernates.
	2777	*
	2778	* @dd Pointer to the driver data structure.
	2779	*
	2780	* return value
	2781	* 0 Suspend was successful
	2782	* -EFAULT Suspend was not successful
	2783	*/
	2784	static int mtip_hw_suspend(struct driver_data *dd)
	2785	{
	2786	/*
	2787	* Send standby immediate (E0h) to the drive
	2788	* so that it saves its state.
	2789	*/
	2790	if (mtip_standby_immediate(dd->port) != 0) {
	2791	dev_err(&dd->pdev->dev,
	2792	"Failed standby-immediate command\n");
	2793	return -EFAULT;
	2794	}
	2795
	2796	/* Disable interrupts on the HBA.*/
	2797	writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
	2798	dd->mmio + HOST_CTL);
	2799	mtip_deinit_port(dd->port);
	2800
	2801	return 0;
	2802	}
	2803
	2804	/*
	2805	* Resume function
	2806	*
	2807	* This function is called by the Block Layer as the
	2808	* system resumes.
	2809	*
	2810	* @dd Pointer to the driver data structure.
	2811	*
	2812	* return value
	2813	* 0 Resume was successful
	2814	* -EFAULT Resume was not successful
	2815	*/
	2816	static int mtip_hw_resume(struct driver_data *dd)
	2817	{
	2818	/* Perform any needed hardware setup steps */
	2819	hba_setup(dd);
	2820
	2821	/* Reset the HBA */
	2822	if (mtip_hba_reset(dd) != 0) {
	2823	dev_err(&dd->pdev->dev,
	2824	"Unable to reset the HBA\n");
	2825	return -EFAULT;
	2826	}
	2827
	2828	/*
	2829	* Enable the port, DMA engine, and FIS reception specific
	2830	* h/w in controller.
	2831	*/
	2832	mtip_init_port(dd->port);
	2833	mtip_start_port(dd->port);
	2834
	2835	/* Enable interrupts on the HBA.*/
	2836	writel(readl(dd->mmio + HOST_CTL) \| HOST_IRQ_EN,
	2837	dd->mmio + HOST_CTL);
	2838
	2839	return 0;
	2840	}
	2841
	2842	/*
	2843	* Helper function for reusing disk name
	2844	* upon hot insertion.
	2845	*/
	2846	static int rssd_disk_name_format(char *prefix,
	2847	int index,
	2848	char *buf,
	2849	int buflen)
	2850	{
	2851	const int base = 'z' - 'a' + 1;
	2852	char *begin = buf + strlen(prefix);
	2853	char *end = buf + buflen;
	2854	char *p;
	2855	int unit;
	2856
	2857	p = end - 1;
	2858	*p = '\0';
	2859	unit = base;
	2860	do {
	2861	if (p == begin)
	2862	return -EINVAL;
	2863	*--p = 'a' + (index % unit);
	2864	index = (index / unit) - 1;
	2865	} while (index >= 0);
	2866
	2867	memmove(begin, p, end - p);
	2868	memcpy(buf, prefix, strlen(prefix));
	2869
	2870	return 0;
	2871	}
	2872
	2873	/*
	2874	* Block layer IOCTL handler.
	2875	*
	2876	* @dev Pointer to the block_device structure.
	2877	* @mode ignored
	2878	* @cmd IOCTL command passed from the user application.
	2879	* @arg Argument passed from the user application.
	2880	*
	2881	* return value
	2882	* 0 IOCTL completed successfully.
	2883	* -ENOTTY IOCTL not supported or invalid driver data
	2884	* structure pointer.
	2885	*/
	2886	static int mtip_block_ioctl(struct block_device *dev,
	2887	fmode_t mode,
	2888	unsigned cmd,
	2889	unsigned long arg)
	2890	{
	2891	struct driver_data *dd = dev->bd_disk->private_data;
	2892
	2893	if (!capable(CAP_SYS_ADMIN))
	2894	return -EACCES;
	2895
	2896	if (!dd)
	2897	return -ENOTTY;
	2898
	2899	switch (cmd) {
	2900	case BLKFLSBUF:
	2901	return -ENOTTY;
	2902	default:
	2903	return mtip_hw_ioctl(dd, cmd, arg);
	2904	}
	2905	}
	2906
	2907	#ifdef CONFIG_COMPAT
	2908	/*
	2909	* Block layer compat IOCTL handler.
	2910	*
	2911	* @dev Pointer to the block_device structure.
	2912	* @mode ignored
	2913	* @cmd IOCTL command passed from the user application.
	2914	* @arg Argument passed from the user application.
	2915	*
	2916	* return value
	2917	* 0 IOCTL completed successfully.
	2918	* -ENOTTY IOCTL not supported or invalid driver data
	2919	* structure pointer.
	2920	*/
	2921	static int mtip_block_compat_ioctl(struct block_device *dev,
	2922	fmode_t mode,
	2923	unsigned cmd,
	2924	unsigned long arg)
	2925	{
	2926	struct driver_data *dd = dev->bd_disk->private_data;
	2927
	2928	if (!capable(CAP_SYS_ADMIN))
	2929	return -EACCES;
	2930
	2931	if (!dd)
	2932	return -ENOTTY;
	2933
	2934	switch (cmd) {
	2935	case BLKFLSBUF:
	2936	return -ENOTTY;
	2937	case HDIO_DRIVE_TASKFILE: {
	2938	struct mtip_compat_ide_task_request_s __user *compat_req_task;
	2939	ide_task_request_t req_task;
	2940	int compat_tasksize, outtotal, ret;
	2941
	2942	compat_tasksize =
	2943	sizeof(struct mtip_compat_ide_task_request_s);
	2944
	2945	compat_req_task =
	2946	(struct mtip_compat_ide_task_request_s __user *) arg;
	2947
	2948	if (copy_from_user(&req_task, (void __user *) arg,
	2949	compat_tasksize - (2 * sizeof(compat_long_t))))
	2950	return -EFAULT;
	2951
	2952	if (get_user(req_task.out_size, &compat_req_task->out_size))
	2953	return -EFAULT;
	2954
	2955	if (get_user(req_task.in_size, &compat_req_task->in_size))
	2956	return -EFAULT;
	2957
	2958	outtotal = sizeof(struct mtip_compat_ide_task_request_s);
	2959
	2960	ret = exec_drive_taskfile(dd, (void __user *) arg,
	2961	&req_task, outtotal);
	2962
	2963	if (copy_to_user((void __user *) arg, &req_task,
	2964	compat_tasksize -
	2965	(2 * sizeof(compat_long_t))))
	2966	return -EFAULT;
	2967
	2968	if (put_user(req_task.out_size, &compat_req_task->out_size))
	2969	return -EFAULT;
	2970
	2971	if (put_user(req_task.in_size, &compat_req_task->in_size))
	2972	return -EFAULT;
	2973
	2974	return ret;
	2975	}
	2976	default:
	2977	return mtip_hw_ioctl(dd, cmd, arg);
	2978	}
	2979	}
	2980	#endif
	2981
	2982	/*
	2983	* Obtain the geometry of the device.
	2984	*
	2985	* You may think that this function is obsolete, but some applications,
	2986	* fdisk for example still used CHS values. This function describes the
	2987	* device as having 224 heads and 56 sectors per cylinder. These values are
	2988	* chosen so that each cylinder is aligned on a 4KB boundary. Since a
	2989	* partition is described in terms of a start and end cylinder this means
	2990	* that each partition is also 4KB aligned. Non-aligned partitions adversely
	2991	* affects performance.
	2992	*
	2993	* @dev Pointer to the block_device strucutre.
	2994	* @geo Pointer to a hd_geometry structure.
	2995	*
	2996	* return value
	2997	* 0 Operation completed successfully.
	2998	* -ENOTTY An error occurred while reading the drive capacity.
	2999	*/
	3000	static int mtip_block_getgeo(struct block_device *dev,
	3001	struct hd_geometry *geo)
	3002	{
	3003	struct driver_data *dd = dev->bd_disk->private_data;
	3004	sector_t capacity;
	3005
	3006	if (!dd)
	3007	return -ENOTTY;
	3008
	3009	if (!(mtip_hw_get_capacity(dd, &capacity))) {
	3010	dev_warn(&dd->pdev->dev,
	3011	"Could not get drive capacity.\n");
	3012	return -ENOTTY;
	3013	}
	3014
	3015	geo->heads = 224;
	3016	geo->sectors = 56;
	3017	sector_div(capacity, (geo->heads * geo->sectors));
	3018	geo->cylinders = capacity;
	3019	return 0;
	3020	}
	3021
	3022	/*
	3023	* Block device operation function.
	3024	*
	3025	* This structure contains pointers to the functions required by the block
	3026	* layer.
	3027	*/
	3028	static const struct block_device_operations mtip_block_ops = {
	3029	.ioctl = mtip_block_ioctl,
	3030	#ifdef CONFIG_COMPAT
	3031	.compat_ioctl = mtip_block_compat_ioctl,
	3032	#endif
	3033	.getgeo = mtip_block_getgeo,
	3034	.owner = THIS_MODULE
	3035	};
	3036
	3037	/*
	3038	* Block layer make request function.
	3039	*
	3040	* This function is called by the kernel to process a BIO for
	3041	* the P320 device.
	3042	*
	3043	* @queue Pointer to the request queue. Unused other than to obtain
	3044	* the driver data structure.
	3045	* @bio Pointer to the BIO.
	3046	*
	3047	*/
	3048	static void mtip_make_request(struct request_queue queue, struct bio bio)
	3049	{
	3050	struct driver_data *dd = queue->queuedata;
	3051	struct scatterlist *sg;
	3052	struct bio_vec *bvec;
	3053	int nents = 0;
	3054	int tag = 0;
	3055
	3056	if (unlikely(!bio_has_data(bio))) {
	3057	blk_queue_flush(queue, 0);
	3058	bio_endio(bio, 0);
	3059	return;
	3060	}
	3061
	3062	sg = mtip_hw_get_scatterlist(dd, &tag);
	3063	if (likely(sg != NULL)) {
	3064	blk_queue_bounce(queue, &bio);
	3065
	3066	if (unlikely((bio)->bi_vcnt > MTIP_MAX_SG)) {
	3067	dev_warn(&dd->pdev->dev,
	3068	"Maximum number of SGL entries exceeded");
	3069	bio_io_error(bio);
	3070	mtip_hw_release_scatterlist(dd, tag);
	3071	return;
	3072	}
	3073
	3074	/* Create the scatter list for this bio. */
	3075	bio_for_each_segment(bvec, bio, nents) {
	3076	sg_set_page(&sg[nents],
	3077	bvec->bv_page,
	3078	bvec->bv_len,
	3079	bvec->bv_offset);
	3080	}
	3081
	3082	/* Issue the read/write. */
	3083	mtip_hw_submit_io(dd,
	3084	bio->bi_sector,
	3085	bio_sectors(bio),
	3086	nents,
	3087	tag,
	3088	bio_endio,
	3089	bio,
	3090	bio->bi_rw & REQ_FUA,
	3091	bio_data_dir(bio));
	3092	} else
	3093	bio_io_error(bio);
	3094	}
	3095
	3096	/*
	3097	* Block layer initialization function.
	3098	*
	3099	* This function is called once by the PCI layer for each P320
	3100	* device that is connected to the system.
	3101	*
	3102	* @dd Pointer to the driver data structure.
	3103	*
	3104	* return value
	3105	* 0 on success else an error code.
	3106	*/
	3107	static int mtip_block_initialize(struct driver_data *dd)
	3108	{
	3109	int rv = 0, wait_for_rebuild = 0;
	3110	sector_t capacity;
	3111	unsigned int index = 0;
	3112	struct kobject *kobj;
	3113	unsigned char thd_name[16];
	3114
	3115	if (dd->disk)
	3116	goto skip_create_disk; /* hw init done, before rebuild */
	3117
	3118	/* Initialize the protocol layer. */
	3119	wait_for_rebuild = mtip_hw_init(dd);
	3120	if (wait_for_rebuild < 0) {
	3121	dev_err(&dd->pdev->dev,
	3122	"Protocol layer initialization failed\n");
	3123	rv = -EINVAL;
	3124	goto protocol_init_error;
	3125	}
	3126
	3127	dd->disk = alloc_disk(MTIP_MAX_MINORS);
	3128	if (dd->disk == NULL) {
	3129	dev_err(&dd->pdev->dev,
	3130	"Unable to allocate gendisk structure\n");
	3131	rv = -EINVAL;
	3132	goto alloc_disk_error;
	3133	}
	3134
	3135	/* Generate the disk name, implemented same as in sd.c */
	3136	do {
	3137	if (!ida_pre_get(&rssd_index_ida, GFP_KERNEL))
	3138	goto ida_get_error;
	3139
	3140	spin_lock(&rssd_index_lock);
	3141	rv = ida_get_new(&rssd_index_ida, &index);
	3142	spin_unlock(&rssd_index_lock);
	3143	} while (rv == -EAGAIN);
	3144
	3145	if (rv)
	3146	goto ida_get_error;
	3147
	3148	rv = rssd_disk_name_format("rssd",
	3149	index,
	3150	dd->disk->disk_name,
	3151	DISK_NAME_LEN);
	3152	if (rv)
	3153	goto disk_index_error;
	3154
	3155	dd->disk->driverfs_dev = &dd->pdev->dev;
	3156	dd->disk->major = dd->major;
	3157	dd->disk->first_minor = dd->instance * MTIP_MAX_MINORS;
	3158	dd->disk->fops = &mtip_block_ops;
	3159	dd->disk->private_data = dd;
	3160	dd->index = index;
	3161
	3162	/*
	3163	* if rebuild pending, start the service thread, and delay the block
	3164	* queue creation and add_disk()
	3165	*/
	3166	if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
	3167	goto start_service_thread;
	3168
	3169	skip_create_disk:
	3170	/* Allocate the request queue. */
	3171	dd->queue = blk_alloc_queue(GFP_KERNEL);
	3172	if (dd->queue == NULL) {
	3173	dev_err(&dd->pdev->dev,
	3174	"Unable to allocate request queue\n");
	3175	rv = -ENOMEM;
	3176	goto block_queue_alloc_init_error;
	3177	}
	3178
	3179	/* Attach our request function to the request queue. */
	3180	blk_queue_make_request(dd->queue, mtip_make_request);
	3181
	3182	dd->disk->queue = dd->queue;
	3183	dd->queue->queuedata = dd;
	3184
	3185	/* Set device limits. */
	3186	set_bit(QUEUE_FLAG_NONROT, &dd->queue->queue_flags);
	3187	blk_queue_max_segments(dd->queue, MTIP_MAX_SG);
	3188	blk_queue_physical_block_size(dd->queue, 4096);
	3189	blk_queue_io_min(dd->queue, 4096);
	3190	blk_queue_flush(dd->queue, 0);
	3191
	3192	/* Set the capacity of the device in 512 byte sectors. */
	3193	if (!(mtip_hw_get_capacity(dd, &capacity))) {
	3194	dev_warn(&dd->pdev->dev,
	3195	"Could not read drive capacity\n");
	3196	rv = -EIO;
	3197	goto read_capacity_error;
	3198	}
	3199	set_capacity(dd->disk, capacity);
	3200
	3201	/* Enable the block device and add it to /dev */
	3202	add_disk(dd->disk);
	3203
	3204	/*
	3205	* Now that the disk is active, initialize any sysfs attributes
	3206	* managed by the protocol layer.
	3207	*/
	3208	kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
	3209	if (kobj) {
	3210	mtip_hw_sysfs_init(dd, kobj);
	3211	kobject_put(kobj);
	3212	}
	3213
	3214	if (dd->mtip_svc_handler)
	3215	return rv; /* service thread created for handling rebuild */
	3216
	3217	start_service_thread:
	3218	sprintf(thd_name, "mtip_svc_thd_%02d", index);
	3219
	3220	dd->mtip_svc_handler = kthread_run(mtip_service_thread,
	3221	dd, thd_name);
	3222
	3223	if (IS_ERR(dd->mtip_svc_handler)) {
	3224	printk(KERN_ERR "mtip32xx: service thread failed to start\n");
	3225	dd->mtip_svc_handler = NULL;
	3226	rv = -EFAULT;
	3227	goto kthread_run_error;
	3228	}
	3229
	3230	return rv;
	3231
	3232	kthread_run_error:
	3233	/* Delete our gendisk. This also removes the device from /dev */
	3234	del_gendisk(dd->disk);
	3235
	3236	read_capacity_error:
	3237	blk_cleanup_queue(dd->queue);
	3238
	3239	block_queue_alloc_init_error:
	3240	disk_index_error:
	3241	spin_lock(&rssd_index_lock);
	3242	ida_remove(&rssd_index_ida, index);
	3243	spin_unlock(&rssd_index_lock);
	3244
	3245	ida_get_error:
	3246	put_disk(dd->disk);
	3247
	3248	alloc_disk_error:
	3249	mtip_hw_exit(dd); /* De-initialize the protocol layer. */
	3250
	3251	protocol_init_error:
	3252	return rv;
	3253	}
	3254
	3255	/*
	3256	* Block layer deinitialization function.
	3257	*
	3258	* Called by the PCI layer as each P320 device is removed.
	3259	*
	3260	* @dd Pointer to the driver data structure.
	3261	*
	3262	* return value
	3263	* 0
	3264	*/
	3265	static int mtip_block_remove(struct driver_data *dd)
	3266	{
	3267	struct kobject *kobj;
	3268
	3269	if (dd->mtip_svc_handler) {
	3270	set_bit(MTIP_FLAG_SVC_THD_SHOULD_STOP_BIT, &dd->port->flags);
	3271	wake_up_interruptible(&dd->port->svc_wait);
	3272	kthread_stop(dd->mtip_svc_handler);
	3273	}
	3274
	3275	/* Clean up the sysfs attributes managed by the protocol layer. */
	3276	kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
	3277	if (kobj) {
	3278	mtip_hw_sysfs_exit(dd, kobj);
	3279	kobject_put(kobj);
	3280	}
	3281
	3282	/*
	3283	* Delete our gendisk structure. This also removes the device
	3284	* from /dev
	3285	*/
	3286	del_gendisk(dd->disk);
	3287	blk_cleanup_queue(dd->queue);
	3288	dd->disk = NULL;
	3289	dd->queue = NULL;
	3290
	3291	/* De-initialize the protocol layer. */
	3292	mtip_hw_exit(dd);
	3293
	3294	return 0;
	3295	}
	3296
	3297	/*
	3298	* Function called by the PCI layer when just before the
	3299	* machine shuts down.
	3300	*
	3301	* If a protocol layer shutdown function is present it will be called
	3302	* by this function.
	3303	*
	3304	* @dd Pointer to the driver data structure.
	3305	*
	3306	* return value
	3307	* 0
	3308	*/
	3309	static int mtip_block_shutdown(struct driver_data *dd)
	3310	{
	3311	dev_info(&dd->pdev->dev,
	3312	"Shutting down %s ...\n", dd->disk->disk_name);
	3313
	3314	/* Delete our gendisk structure, and cleanup the blk queue. */
	3315	del_gendisk(dd->disk);
	3316	blk_cleanup_queue(dd->queue);
	3317	dd->disk = NULL;
	3318	dd->queue = NULL;
	3319
	3320	mtip_hw_shutdown(dd);
	3321	return 0;
	3322	}
	3323
	3324	static int mtip_block_suspend(struct driver_data *dd)
	3325	{
	3326	dev_info(&dd->pdev->dev,
	3327	"Suspending %s ...\n", dd->disk->disk_name);
	3328	mtip_hw_suspend(dd);
	3329	return 0;
	3330	}
	3331
	3332	static int mtip_block_resume(struct driver_data *dd)
	3333	{
	3334	dev_info(&dd->pdev->dev, "Resuming %s ...\n",
	3335	dd->disk->disk_name);
	3336	mtip_hw_resume(dd);
	3337	return 0;
	3338	}
	3339
	3340	/*
	3341	* Called for each supported PCI device detected.
	3342	*
	3343	* This function allocates the private data structure, enables the
	3344	* PCI device and then calls the block layer initialization function.
	3345	*
	3346	* return value
	3347	* 0 on success else an error code.
	3348	*/
	3349	static int mtip_pci_probe(struct pci_dev *pdev,
	3350	const struct pci_device_id *ent)
	3351	{
	3352	int rv = 0;
	3353	struct driver_data *dd = NULL;
	3354
	3355	/* Allocate memory for this devices private data. */
	3356	dd = kzalloc(sizeof(struct driver_data), GFP_KERNEL);
	3357	if (dd == NULL) {
	3358	dev_err(&pdev->dev,
	3359	"Unable to allocate memory for driver data\n");
	3360	return -ENOMEM;
	3361	}
	3362
	3363	/* Set the atomic variable as 1 in case of SRSI */
	3364	atomic_set(&dd->drv_cleanup_done, true);
	3365
	3366	atomic_set(&dd->resumeflag, false);
	3367
	3368	/* Attach the private data to this PCI device. */
	3369	pci_set_drvdata(pdev, dd);
	3370
	3371	rv = pcim_enable_device(pdev);
	3372	if (rv < 0) {
	3373	dev_err(&pdev->dev, "Unable to enable device\n");
	3374	goto iomap_err;
	3375	}
	3376
	3377	/* Map BAR5 to memory. */
	3378	rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME);
	3379	if (rv < 0) {
	3380	dev_err(&pdev->dev, "Unable to map regions\n");
	3381	goto iomap_err;
	3382	}
	3383
	3384	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
	3385	rv = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	3386
	3387	if (rv) {
	3388	rv = pci_set_consistent_dma_mask(pdev,
	3389	DMA_BIT_MASK(32));
	3390	if (rv) {
	3391	dev_warn(&pdev->dev,
	3392	"64-bit DMA enable failed\n");
	3393	goto setmask_err;
	3394	}
	3395	}
	3396	}
	3397
	3398	pci_set_master(pdev);
	3399
	3400	if (pci_enable_msi(pdev)) {
	3401	dev_warn(&pdev->dev,
	3402	"Unable to enable MSI interrupt.\n");
	3403	goto block_initialize_err;
	3404	}
	3405
	3406	/* Copy the info we may need later into the private data structure. */
	3407	dd->major = mtip_major;
	3408	dd->instance = instance;
	3409	dd->pdev = pdev;
	3410
	3411	/* Initialize the block layer. */
	3412	rv = mtip_block_initialize(dd);
	3413	if (rv < 0) {
	3414	dev_err(&pdev->dev,
	3415	"Unable to initialize block layer\n");
	3416	goto block_initialize_err;
	3417	}
	3418
	3419	/*
	3420	* Increment the instance count so that each device has a unique
	3421	* instance number.
	3422	*/
	3423	instance++;
	3424
	3425	goto done;
	3426
	3427	block_initialize_err:
	3428	pci_disable_msi(pdev);
	3429
	3430	setmask_err:
	3431	pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
	3432
	3433	iomap_err:
	3434	kfree(dd);
	3435	pci_set_drvdata(pdev, NULL);
	3436	return rv;
	3437	done:
	3438	/* Set the atomic variable as 0 in case of SRSI */
	3439	atomic_set(&dd->drv_cleanup_done, true);
	3440
	3441	return rv;
	3442	}
	3443
	3444	/*
	3445	* Called for each probed device when the device is removed or the
	3446	* driver is unloaded.
	3447	*
	3448	* return value
	3449	* None
	3450	*/
	3451	static void mtip_pci_remove(struct pci_dev *pdev)
	3452	{
	3453	struct driver_data *dd = pci_get_drvdata(pdev);
	3454	int counter = 0;
	3455
	3456	if (mtip_check_surprise_removal(pdev)) {
	3457	while (atomic_read(&dd->drv_cleanup_done) == false) {
	3458	counter++;
	3459	msleep(20);
	3460	if (counter == 10) {
	3461	/* Cleanup the outstanding commands */
	3462	mtip_command_cleanup(dd);
	3463	break;
	3464	}
	3465	}
	3466	}
	3467	/* Set the atomic variable as 1 in case of SRSI */
	3468	atomic_set(&dd->drv_cleanup_done, true);
	3469
	3470	/* Clean up the block layer. */
	3471	mtip_block_remove(dd);
	3472
	3473	pci_disable_msi(pdev);
	3474
	3475	kfree(dd);
	3476	pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
	3477	}
	3478
	3479	/*
	3480	* Called for each probed device when the device is suspended.
	3481	*
	3482	* return value
	3483	* 0 Success
	3484	* <0 Error
	3485	*/
	3486	static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg)
	3487	{
	3488	int rv = 0;
	3489	struct driver_data *dd = pci_get_drvdata(pdev);
	3490
	3491	if (!dd) {
	3492	dev_err(&pdev->dev,
	3493	"Driver private datastructure is NULL\n");
	3494	return -EFAULT;
	3495	}
	3496
	3497	atomic_set(&dd->resumeflag, true);
	3498
	3499	/* Disable ports & interrupts then send standby immediate */
	3500	rv = mtip_block_suspend(dd);
	3501	if (rv < 0) {
	3502	dev_err(&pdev->dev,
	3503	"Failed to suspend controller\n");
	3504	return rv;
	3505	}
	3506
	3507	/*
	3508	* Save the pci config space to pdev structure &
	3509	* disable the device
	3510	*/
	3511	pci_save_state(pdev);
	3512	pci_disable_device(pdev);
	3513
	3514	/* Move to Low power state*/
	3515	pci_set_power_state(pdev, PCI_D3hot);
	3516
	3517	return rv;
	3518	}
	3519
	3520	/*
	3521	* Called for each probed device when the device is resumed.
	3522	*
	3523	* return value
	3524	* 0 Success
	3525	* <0 Error
	3526	*/
	3527	static int mtip_pci_resume(struct pci_dev *pdev)
	3528	{
	3529	int rv = 0;
	3530	struct driver_data *dd;
	3531
	3532	dd = pci_get_drvdata(pdev);
	3533	if (!dd) {
	3534	dev_err(&pdev->dev,
	3535	"Driver private datastructure is NULL\n");
	3536	return -EFAULT;
	3537	}
	3538
	3539	/* Move the device to active State */
	3540	pci_set_power_state(pdev, PCI_D0);
	3541
	3542	/* Restore PCI configuration space */
	3543	pci_restore_state(pdev);
	3544
	3545	/* Enable the PCI device*/
	3546	rv = pcim_enable_device(pdev);
	3547	if (rv < 0) {
	3548	dev_err(&pdev->dev,
	3549	"Failed to enable card during resume\n");
	3550	goto err;
	3551	}
	3552	pci_set_master(pdev);
	3553
	3554	/*
	3555	* Calls hbaReset, initPort, & startPort function
	3556	* then enables interrupts
	3557	*/
	3558	rv = mtip_block_resume(dd);
	3559	if (rv < 0)
	3560	dev_err(&pdev->dev, "Unable to resume\n");
	3561
	3562	err:
	3563	atomic_set(&dd->resumeflag, false);
	3564
	3565	return rv;
	3566	}
	3567
	3568	/*
	3569	* Shutdown routine
	3570	*
	3571	* return value
	3572	* None
	3573	*/
	3574	static void mtip_pci_shutdown(struct pci_dev *pdev)
	3575	{
	3576	struct driver_data *dd = pci_get_drvdata(pdev);
	3577	if (dd)
	3578	mtip_block_shutdown(dd);
	3579	}
	3580
	3581	/* Table of device ids supported by this driver. */
	3582	static DEFINE_PCI_DEVICE_TABLE(mtip_pci_tbl) = {
	3583	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320_DEVICE_ID) },
	3584	{ 0 }
	3585	};
	3586
	3587	/* Structure that describes the PCI driver functions. */
	3588	static struct pci_driver mtip_pci_driver = {
	3589	.name = MTIP_DRV_NAME,
	3590	.id_table = mtip_pci_tbl,
	3591	.probe = mtip_pci_probe,
	3592	.remove = mtip_pci_remove,
	3593	.suspend = mtip_pci_suspend,
	3594	.resume = mtip_pci_resume,
	3595	.shutdown = mtip_pci_shutdown,
	3596	};
	3597
	3598	MODULE_DEVICE_TABLE(pci, mtip_pci_tbl);
	3599
	3600	/*
	3601	* Module initialization function.
	3602	*
	3603	* Called once when the module is loaded. This function allocates a major
	3604	* block device number to the Cyclone devices and registers the PCI layer
	3605	* of the driver.
	3606	*
	3607	* Return value
	3608	* 0 on success else error code.
	3609	*/
	3610	static int __init mtip_init(void)
	3611	{
	3612	printk(KERN_INFO MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n");
	3613
	3614	/* Allocate a major block device number to use with this driver. */
	3615	mtip_major = register_blkdev(0, MTIP_DRV_NAME);
	3616	if (mtip_major < 0) {
	3617	printk(KERN_ERR "Unable to register block device (%d)\n",
	3618	mtip_major);
	3619	return -EBUSY;
	3620	}
	3621
	3622	/* Register our PCI operations. */
	3623	return pci_register_driver(&mtip_pci_driver);
	3624	}
	3625
	3626	/*
	3627	* Module de-initialization function.
	3628	*
	3629	* Called once when the module is unloaded. This function deallocates
	3630	* the major block device number allocated by mtip_init() and
	3631	* unregisters the PCI layer of the driver.
	3632	*
	3633	* Return value
	3634	* none
	3635	*/
	3636	static void __exit mtip_exit(void)
	3637	{
	3638	/* Release the allocated major block device number. */
	3639	unregister_blkdev(mtip_major, MTIP_DRV_NAME);
	3640
	3641	/* Unregister the PCI driver. */
	3642	pci_unregister_driver(&mtip_pci_driver);
	3643	}
	3644
	3645	MODULE_AUTHOR("Micron Technology, Inc");
	3646	MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver");
	3647	MODULE_LICENSE("GPL");
	3648	MODULE_VERSION(MTIP_DRV_VERSION);
	3649
	3650	module_init(mtip_init);
	3651	module_exit(mtip_exit);
	3652

drivers/block/mtip32xx/mtip32xx.h

Diff comments View file @ 16008d6

File was created	1	/*
	2	* mtip32xx.h - Header file for the P320 SSD Block Driver
	3	* Copyright (C) 2011 Micron Technology, Inc.
	4	*
	5	* Portions of this code were derived from works subjected to the
	6	* following copyright:
	7	* Copyright (C) 2009 Integrated Device Technology, Inc.
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of the GNU General Public License as published by
	11	* the Free Software Foundation; either version 2 of the License, or
	12	* (at your option) any later version.
	13	*
	14	* This program is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	17	* GNU General Public License for more details.
	18	*
	19	*/
	20
	21	#ifndef __MTIP32XX_H__
	22	#define __MTIP32XX_H__
	23
	24	#include <linux/spinlock.h>
	25	#include <linux/rwsem.h>
	26	#include <linux/ata.h>
	27	#include <linux/interrupt.h>
	28	#include <linux/genhd.h>
	29	#include <linux/version.h>
	30
	31	/* Offset of Subsystem Device ID in pci confoguration space */
	32	#define PCI_SUBSYSTEM_DEVICEID 0x2E
	33
	34	/* offset of Device Control register in PCIe extended capabilites space */
	35	#define PCIE_CONFIG_EXT_DEVICE_CONTROL_OFFSET 0x48
	36
	37	/* # of times to retry timed out IOs */
	38	#define MTIP_MAX_RETRIES 5
	39
	40	/* Various timeout values in ms */
	41	#define MTIP_NCQ_COMMAND_TIMEOUT_MS 5000
	42	#define MTIP_IOCTL_COMMAND_TIMEOUT_MS 5000
	43	#define MTIP_INTERNAL_COMMAND_TIMEOUT_MS 5000
	44
	45	/* check for timeouts every 500ms */
	46	#define MTIP_TIMEOUT_CHECK_PERIOD 500
	47
	48	/* ftl rebuild */
	49	#define MTIP_FTL_REBUILD_OFFSET 142
	50	#define MTIP_FTL_REBUILD_MAGIC 0xED51
	51	#define MTIP_FTL_REBUILD_TIMEOUT_MS 2400000
	52
	53	/* Macro to extract the tag bit number from a tag value. */
	54	#define MTIP_TAG_BIT(tag) (tag & 0x1F)
	55
	56	/*
	57	* Macro to extract the tag index from a tag value. The index
	58	* is used to access the correct s_active/Command Issue register based
	59	* on the tag value.
	60	*/
	61	#define MTIP_TAG_INDEX(tag) (tag >> 5)
	62
	63	/*
	64	* Maximum number of scatter gather entries
	65	* a single command may have.
	66	*/
	67	#define MTIP_MAX_SG 128
	68
	69	/*
	70	* Maximum number of slot groups (Command Issue & s_active registers)
	71	* NOTE: This is the driver maximum; check dd->slot_groups for actual value.
	72	*/
	73	#define MTIP_MAX_SLOT_GROUPS 8
	74
	75	/* Internal command tag. */
	76	#define MTIP_TAG_INTERNAL 0
	77
	78	/* Micron Vendor ID & P320x SSD Device ID */
	79	#define PCI_VENDOR_ID_MICRON 0x1344
	80	#define P320_DEVICE_ID 0x5150
	81
	82	/* Driver name and version strings */
	83	#define MTIP_DRV_NAME "mtip32xx"
	84	#define MTIP_DRV_VERSION "1.2.6os3"
	85
	86	/* Maximum number of minor device numbers per device. */
	87	#define MTIP_MAX_MINORS 16
	88
	89	/* Maximum number of supported command slots. */
	90	#define MTIP_MAX_COMMAND_SLOTS (MTIP_MAX_SLOT_GROUPS * 32)
	91
	92	/*
	93	* Per-tag bitfield size in longs.
	94	* Linux bit manipulation functions
	95	* (i.e. test_and_set_bit, find_next_zero_bit)
	96	* manipulate memory in longs, so we try to make the math work.
	97	* take the slot groups and find the number of longs, rounding up.
	98	* Careful! i386 and x86_64 use different size longs!
	99	*/
	100	#define U32_PER_LONG (sizeof(long) / sizeof(u32))
	101	#define SLOTBITS_IN_LONGS ((MTIP_MAX_SLOT_GROUPS + \
	102	(U32_PER_LONG-1))/U32_PER_LONG)
	103
	104	/* BAR number used to access the HBA registers. */
	105	#define MTIP_ABAR 5
	106
	107	/* Forced Unit Access Bit */
	108	#define FUA_BIT 0x80
	109
	110	#ifdef DEBUG
	111	#define dbg_printk(format, arg...) \
	112	printk(pr_fmt(format), ##arg);
	113	#else
	114	#define dbg_printk(format, arg...)
	115	#endif
	116
	117	#define __force_bit2int (unsigned int __force)
	118
	119	/* below are bit numbers in 'flags' defined in mtip_port */
	120	#define MTIP_FLAG_IC_ACTIVE_BIT 0
	121	#define MTIP_FLAG_EH_ACTIVE_BIT 1
	122	#define MTIP_FLAG_SVC_THD_ACTIVE_BIT 2
	123	#define MTIP_FLAG_ISSUE_CMDS_BIT 4
	124	#define MTIP_FLAG_REBUILD_BIT 5
	125	#define MTIP_FLAG_SVC_THD_SHOULD_STOP_BIT 8
	126
	127	/* Register Frame Information Structure (FIS), host to device. */
	128	struct host_to_dev_fis {
	129	/*
	130	* FIS type.
	131	* - 27h Register FIS, host to device.
	132	* - 34h Register FIS, device to host.
	133	* - 39h DMA Activate FIS, device to host.
	134	* - 41h DMA Setup FIS, bi-directional.
	135	* - 46h Data FIS, bi-directional.
	136	* - 58h BIST Activate FIS, bi-directional.
	137	* - 5Fh PIO Setup FIS, device to host.
	138	* - A1h Set Device Bits FIS, device to host.
	139	*/
	140	unsigned char type;
	141	unsigned char opts;
	142	unsigned char command;
	143	unsigned char features;
	144
	145	union {
	146	unsigned char lba_low;
	147	unsigned char sector;
	148	};
	149	union {
	150	unsigned char lba_mid;
	151	unsigned char cyl_low;
	152	};
	153	union {
	154	unsigned char lba_hi;
	155	unsigned char cyl_hi;
	156	};
	157	union {
	158	unsigned char device;
	159	unsigned char head;
	160	};
	161
	162	union {
	163	unsigned char lba_low_ex;
	164	unsigned char sector_ex;
	165	};
	166	union {
	167	unsigned char lba_mid_ex;
	168	unsigned char cyl_low_ex;
	169	};
	170	union {
	171	unsigned char lba_hi_ex;
	172	unsigned char cyl_hi_ex;
	173	};
	174	unsigned char features_ex;
	175
	176	unsigned char sect_count;
	177	unsigned char sect_cnt_ex;
	178	unsigned char res2;
	179	unsigned char control;
	180
	181	unsigned int res3;
	182	};
	183
	184	/* Command header structure. */
	185	struct mtip_cmd_hdr {
	186	/*
	187	* Command options.
	188	* - Bits 31:16 Number of PRD entries.
	189	* - Bits 15:8 Unused in this implementation.
	190	* - Bit 7 Prefetch bit, informs the drive to prefetch PRD entries.
	191	* - Bit 6 Write bit, should be set when writing data to the device.
	192	* - Bit 5 Unused in this implementation.
	193	* - Bits 4:0 Length of the command FIS in DWords (DWord = 4 bytes).
	194	*/
	195	unsigned int opts;
	196	/* This field is unsed when using NCQ. */
	197	union {
	198	unsigned int byte_count;
	199	unsigned int status;
	200	};
	201	/*
	202	* Lower 32 bits of the command table address associated with this
	203	* header. The command table addresses must be 128 byte aligned.
	204	*/
	205	unsigned int ctba;
	206	/*
	207	* If 64 bit addressing is used this field is the upper 32 bits
	208	* of the command table address associated with this command.
	209	*/
	210	unsigned int ctbau;
	211	/* Reserved and unused. */
	212	unsigned int res[4];
	213	};
	214
	215	/* Command scatter gather structure (PRD). */
	216	struct mtip_cmd_sg {
	217	/*
	218	* Low 32 bits of the data buffer address. For P320 this
	219	* address must be 8 byte aligned signified by bits 2:0 being
	220	* set to 0.
	221	*/
	222	unsigned int dba;
	223	/*
	224	* When 64 bit addressing is used this field is the upper
	225	* 32 bits of the data buffer address.
	226	*/
	227	unsigned int dba_upper;
	228	/* Unused. */
	229	unsigned int reserved;
	230	/*
	231	* Bit 31: interrupt when this data block has been transferred.
	232	* Bits 30..22: reserved
	233	* Bits 21..0: byte count (minus 1). For P320 the byte count must be
	234	* 8 byte aligned signified by bits 2:0 being set to 1.
	235	*/
	236	unsigned int info;
	237	};
	238	struct mtip_port;
	239
	240	/* Structure used to describe a command. */
	241	struct mtip_cmd {
	242
	243	struct mtip_cmd_hdr command_header; / ptr to command header entry */
	244
	245	dma_addr_t command_header_dma; /* corresponding physical address */
	246
	247	void command; / ptr to command table entry */
	248
	249	dma_addr_t command_dma; /* corresponding physical address */
	250
	251	void comp_data; / data passed to completion function comp_func() */
	252	/*
	253	* Completion function called by the ISR upon completion of
	254	* a command.
	255	*/
	256	void (comp_func)(struct mtip_port port,
	257	int tag,
	258	void *data,
	259	int status);
	260	/* Additional callback function that may be called by comp_func() */
	261	void (async_callback)(void data, int status);
	262
	263	void async_data; / Addl. data passed to async_callback() */
	264
	265	int scatter_ents; /* Number of scatter list entries used */
	266
	267	struct scatterlist sg[MTIP_MAX_SG]; /* Scatter list entries */
	268
	269	int retries; /* The number of retries left for this command. */
	270
	271	int direction; /* Data transfer direction */
	272
	273	unsigned long comp_time; /* command completion time, in jiffies */
	274
	275	atomic_t active; /* declares if this command sent to the drive. */
	276	};
	277
	278	/* Structure used to describe a port. */
	279	struct mtip_port {
	280	/* Pointer back to the driver data for this port. */
	281	struct driver_data *dd;
	282	/*
	283	* Used to determine if the data pointed to by the
	284	* identify field is valid.
	285	*/
	286	unsigned long identify_valid;
	287	/* Base address of the memory mapped IO for the port. */
	288	void __iomem *mmio;
	289	/* Array of pointers to the memory mapped s_active registers. */
	290	void __iomem *s_active[MTIP_MAX_SLOT_GROUPS];
	291	/* Array of pointers to the memory mapped completed registers. */
	292	void __iomem *completed[MTIP_MAX_SLOT_GROUPS];
	293	/* Array of pointers to the memory mapped Command Issue registers. */
	294	void __iomem *cmd_issue[MTIP_MAX_SLOT_GROUPS];
	295	/*
	296	* Pointer to the beginning of the command header memory as used
	297	* by the driver.
	298	*/
	299	void *command_list;
	300	/*
	301	* Pointer to the beginning of the command header memory as used
	302	* by the DMA.
	303	*/
	304	dma_addr_t command_list_dma;
	305	/*
	306	* Pointer to the beginning of the RX FIS memory as used
	307	* by the driver.
	308	*/
	309	void *rxfis;
	310	/*
	311	* Pointer to the beginning of the RX FIS memory as used
	312	* by the DMA.
	313	*/
	314	dma_addr_t rxfis_dma;
	315	/*
	316	* Pointer to the beginning of the command table memory as used
	317	* by the driver.
	318	*/
	319	void *command_table;
	320	/*
	321	* Pointer to the beginning of the command table memory as used
	322	* by the DMA.
	323	*/
	324	dma_addr_t command_tbl_dma;
	325	/*
	326	* Pointer to the beginning of the identify data memory as used
	327	* by the driver.
	328	*/
	329	u16 *identify;
	330	/*
	331	* Pointer to the beginning of the identify data memory as used
	332	* by the DMA.
	333	*/
	334	dma_addr_t identify_dma;
	335	/*
	336	* Pointer to the beginning of a sector buffer that is used
	337	* by the driver when issuing internal commands.
	338	*/
	339	u16 *sector_buffer;
	340	/*
	341	* Pointer to the beginning of a sector buffer that is used
	342	* by the DMA when the driver issues internal commands.
	343	*/
	344	dma_addr_t sector_buffer_dma;
	345	/*
	346	* Bit significant, used to determine if a command slot has
	347	* been allocated. i.e. the slot is in use. Bits are cleared
	348	* when the command slot and all associated data structures
	349	* are no longer needed.
	350	*/
	351	unsigned long allocated[SLOTBITS_IN_LONGS];
	352	/*
	353	* used to queue commands when an internal command is in progress
	354	* or error handling is active
	355	*/
	356	unsigned long cmds_to_issue[SLOTBITS_IN_LONGS];
	357	/*
	358	* Array of command slots. Structure includes pointers to the
	359	* command header and command table, and completion function and data
	360	* pointers.
	361	*/
	362	struct mtip_cmd commands[MTIP_MAX_COMMAND_SLOTS];
	363	/* Used by mtip_service_thread to wait for an event */
	364	wait_queue_head_t svc_wait;
	365	/*
	366	* indicates the state of the port. Also, helps the service thread
	367	* to determine its action on wake up.
	368	*/
	369	unsigned long flags;
	370	/*
	371	* Timer used to complete commands that have been active for too long.
	372	*/
	373	struct timer_list cmd_timer;
	374	/*
	375	* Semaphore used to block threads if there are no
	376	* command slots available.
	377	*/
	378	struct semaphore cmd_slot;
	379	/* Spinlock for working around command-issue bug. */
	380	spinlock_t cmd_issue_lock;
	381	};
	382
	383	/*
	384	* Driver private data structure.
	385	*
	386	* One structure is allocated per probed device.
	387	*/
	388	struct driver_data {
	389	void __iomem mmio; / Base address of the HBA registers. */
	390
	391	int major; /* Major device number. */
	392
	393	int instance; /* Instance number. First device probed is 0, ... */
	394
	395	struct gendisk disk; / Pointer to our gendisk structure. */
	396
	397	struct pci_dev pdev; / Pointer to the PCI device structure. */
	398
	399	struct request_queue queue; / Our request queue. */
	400
	401	struct mtip_port port; / Pointer to the port data structure. */
	402
	403	/* Tasklet used to process the bottom half of the ISR. */
	404	struct tasklet_struct tasklet;
	405
	406	unsigned product_type; /* magic value declaring the product type */
	407
	408	unsigned slot_groups; /* number of slot groups the product supports */
	409
	410	atomic_t drv_cleanup_done; /* Atomic variable for SRSI */
	411
	412	unsigned long index; /* Index to determine the disk name */
	413
	414	unsigned int ftlrebuildflag; /* FTL rebuild flag */
	415
	416	atomic_t resumeflag; /* Atomic variable to track suspend/resume */
	417
	418	atomic_t eh_active; /* Flag for error handling tracking */
	419
	420	struct task_struct mtip_svc_handler; / task_struct of svc thd */
	421	};
	422
	423	#endif
	424

drivers/block/xen-blkback/blkback.c

Diff comments View file @ 16008d6

1	/******************************************************************************	1	/******************************************************************************
2	*	2	*
3	* Back-end of the driver for virtual block devices. This portion of the	3	* Back-end of the driver for virtual block devices. This portion of the
4	* driver exports a 'unified' block-device interface that can be accessed	4	* driver exports a 'unified' block-device interface that can be accessed
5	* by any operating system that implements a compatible front end. A	5	* by any operating system that implements a compatible front end. A
6	* reference front-end implementation can be found in:	6	* reference front-end implementation can be found in:
7	* drivers/block/xen-blkfront.c	7	* drivers/block/xen-blkfront.c
8	*	8	*
9	* Copyright (c) 2003-2004, Keir Fraser & Steve Hand	9	* Copyright (c) 2003-2004, Keir Fraser & Steve Hand
10	* Copyright (c) 2005, Christopher Clark	10	* Copyright (c) 2005, Christopher Clark
11	*	11	*
12	* This program is free software; you can redistribute it and/or	12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License version 2	13	* modify it under the terms of the GNU General Public License version 2
14	* as published by the Free Software Foundation; or, when distributed	14	* as published by the Free Software Foundation; or, when distributed
15	* separately from the Linux kernel or incorporated into other	15	* separately from the Linux kernel or incorporated into other
16	* software packages, subject to the following license:	16	* software packages, subject to the following license:
17	*	17	*
18	* Permission is hereby granted, free of charge, to any person obtaining a copy	18	* Permission is hereby granted, free of charge, to any person obtaining a copy
19	* of this source file (the "Software"), to deal in the Software without	19	* of this source file (the "Software"), to deal in the Software without
20	* restriction, including without limitation the rights to use, copy, modify,	20	* restriction, including without limitation the rights to use, copy, modify,
21	* merge, publish, distribute, sublicense, and/or sell copies of the Software,	21	* merge, publish, distribute, sublicense, and/or sell copies of the Software,
22	* and to permit persons to whom the Software is furnished to do so, subject to	22	* and to permit persons to whom the Software is furnished to do so, subject to
23	* the following conditions:	23	* the following conditions:
24	*	24	*
25	* The above copyright notice and this permission notice shall be included in	25	* The above copyright notice and this permission notice shall be included in
26	* all copies or substantial portions of the Software.	26	* all copies or substantial portions of the Software.
27	*	27	*
28	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR	28	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,	29	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE	30	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER	31	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING	32	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
33	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS	33	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34	* IN THE SOFTWARE.	34	* IN THE SOFTWARE.
35	*/	35	*/
36		36
37	#include <linux/spinlock.h>	37	#include <linux/spinlock.h>
38	#include <linux/kthread.h>	38	#include <linux/kthread.h>
39	#include <linux/list.h>	39	#include <linux/list.h>
40	#include <linux/delay.h>	40	#include <linux/delay.h>
41	#include <linux/freezer.h>	41	#include <linux/freezer.h>
42	#include <linux/loop.h>
43	#include <linux/falloc.h>
44	#include <linux/fs.h>
45		42
46	#include <xen/events.h>	43	#include <xen/events.h>
47	#include <xen/page.h>	44	#include <xen/page.h>
48	#include <asm/xen/hypervisor.h>	45	#include <asm/xen/hypervisor.h>
49	#include <asm/xen/hypercall.h>	46	#include <asm/xen/hypercall.h>
50	#include "common.h"	47	#include "common.h"
51		48
52	/*	49	/*
53	* These are rather arbitrary. They are fairly large because adjacent requests	50	* These are rather arbitrary. They are fairly large because adjacent requests
54	* pulled from a communication ring are quite likely to end up being part of	51	* pulled from a communication ring are quite likely to end up being part of
55	* the same scatter/gather request at the disc.	52	* the same scatter/gather request at the disc.
56	*	53	*
57	* TRY INCREASING 'xen_blkif_reqs' IF WRITE SPEEDS SEEM TOO LOW	54	* TRY INCREASING 'xen_blkif_reqs' IF WRITE SPEEDS SEEM TOO LOW
58	*	55	*
59	* This will increase the chances of being able to write whole tracks.	56	* This will increase the chances of being able to write whole tracks.
60	* 64 should be enough to keep us competitive with Linux.	57	* 64 should be enough to keep us competitive with Linux.
61	*/	58	*/
62	static int xen_blkif_reqs = 64;	59	static int xen_blkif_reqs = 64;
63	module_param_named(reqs, xen_blkif_reqs, int, 0);	60	module_param_named(reqs, xen_blkif_reqs, int, 0);
64	MODULE_PARM_DESC(reqs, "Number of blkback requests to allocate");	61	MODULE_PARM_DESC(reqs, "Number of blkback requests to allocate");
65		62
66	/* Run-time switchable: /sys/module/blkback/parameters/ */	63	/* Run-time switchable: /sys/module/blkback/parameters/ */
67	static unsigned int log_stats;	64	static unsigned int log_stats;
68	module_param(log_stats, int, 0644);	65	module_param(log_stats, int, 0644);
69		66
70	/*	67	/*
71	* Each outstanding request that we've passed to the lower device layers has a	68	* Each outstanding request that we've passed to the lower device layers has a
72	* 'pending_req' allocated to it. Each buffer_head that completes decrements	69	* 'pending_req' allocated to it. Each buffer_head that completes decrements
73	* the pendcnt towards zero. When it hits zero, the specified domain has a	70	* the pendcnt towards zero. When it hits zero, the specified domain has a
74	* response queued for it, with the saved 'id' passed back.	71	* response queued for it, with the saved 'id' passed back.
75	*/	72	*/
76	struct pending_req {	73	struct pending_req {
77	struct xen_blkif *blkif;	74	struct xen_blkif *blkif;
78	u64 id;	75	u64 id;
79	int nr_pages;	76	int nr_pages;
80	atomic_t pendcnt;	77	atomic_t pendcnt;
81	unsigned short operation;	78	unsigned short operation;
82	int status;	79	int status;
83	struct list_head free_list;	80	struct list_head free_list;
84	};	81	};
85		82
86	#define BLKBACK_INVALID_HANDLE (~0)	83	#define BLKBACK_INVALID_HANDLE (~0)
87		84
88	struct xen_blkbk {	85	struct xen_blkbk {
89	struct pending_req *pending_reqs;	86	struct pending_req *pending_reqs;
90	/* List of all 'pending_req' available */	87	/* List of all 'pending_req' available */
91	struct list_head pending_free;	88	struct list_head pending_free;
92	/* And its spinlock. */	89	/* And its spinlock. */
93	spinlock_t pending_free_lock;	90	spinlock_t pending_free_lock;
94	wait_queue_head_t pending_free_wq;	91	wait_queue_head_t pending_free_wq;
95	/* The list of all pages that are available. */	92	/* The list of all pages that are available. */
96	struct page **pending_pages;	93	struct page **pending_pages;
97	/* And the grant handles that are available. */	94	/* And the grant handles that are available. */
98	grant_handle_t *pending_grant_handles;	95	grant_handle_t *pending_grant_handles;
99	};	96	};
100		97
101	static struct xen_blkbk *blkbk;	98	static struct xen_blkbk *blkbk;
102		99
103	/*	100	/*
104	* Little helpful macro to figure out the index and virtual address of the	101	* Little helpful macro to figure out the index and virtual address of the
105	* pending_pages[..]. For each 'pending_req' we have have up to	102	* pending_pages[..]. For each 'pending_req' we have have up to
106	* BLKIF_MAX_SEGMENTS_PER_REQUEST (11) pages. The seg would be from 0 through	103	* BLKIF_MAX_SEGMENTS_PER_REQUEST (11) pages. The seg would be from 0 through
107	* 10 and would index in the pending_pages[..].	104	* 10 and would index in the pending_pages[..].
108	*/	105	*/
109	static inline int vaddr_pagenr(struct pending_req *req, int seg)	106	static inline int vaddr_pagenr(struct pending_req *req, int seg)
110	{	107	{
111	return (req - blkbk->pending_reqs) *	108	return (req - blkbk->pending_reqs) *
112	BLKIF_MAX_SEGMENTS_PER_REQUEST + seg;	109	BLKIF_MAX_SEGMENTS_PER_REQUEST + seg;
113	}	110	}
114		111
115	#define pending_page(req, seg) pending_pages[vaddr_pagenr(req, seg)]	112	#define pending_page(req, seg) pending_pages[vaddr_pagenr(req, seg)]
116		113
117	static inline unsigned long vaddr(struct pending_req *req, int seg)	114	static inline unsigned long vaddr(struct pending_req *req, int seg)
118	{	115	{
119	unsigned long pfn = page_to_pfn(blkbk->pending_page(req, seg));	116	unsigned long pfn = page_to_pfn(blkbk->pending_page(req, seg));
120	return (unsigned long)pfn_to_kaddr(pfn);	117	return (unsigned long)pfn_to_kaddr(pfn);
121	}	118	}
122		119
123	#define pending_handle(_req, _seg) \	120	#define pending_handle(_req, _seg) \
124	(blkbk->pending_grant_handles[vaddr_pagenr(_req, _seg)])	121	(blkbk->pending_grant_handles[vaddr_pagenr(_req, _seg)])
125		122
126		123
127	static int do_block_io_op(struct xen_blkif *blkif);	124	static int do_block_io_op(struct xen_blkif *blkif);
128	static int dispatch_rw_block_io(struct xen_blkif *blkif,	125	static int dispatch_rw_block_io(struct xen_blkif *blkif,
129	struct blkif_request *req,	126	struct blkif_request *req,
130	struct pending_req *pending_req);	127	struct pending_req *pending_req);
131	static void make_response(struct xen_blkif *blkif, u64 id,	128	static void make_response(struct xen_blkif *blkif, u64 id,
132	unsigned short op, int st);	129	unsigned short op, int st);
133		130
134	/*	131	/*
135	* Retrieve from the 'pending_reqs' a free pending_req structure to be used.	132	* Retrieve from the 'pending_reqs' a free pending_req structure to be used.
136	*/	133	*/
137	static struct pending_req *alloc_req(void)	134	static struct pending_req *alloc_req(void)
138	{	135	{
139	struct pending_req *req = NULL;	136	struct pending_req *req = NULL;
140	unsigned long flags;	137	unsigned long flags;
141		138
142	spin_lock_irqsave(&blkbk->pending_free_lock, flags);	139	spin_lock_irqsave(&blkbk->pending_free_lock, flags);
143	if (!list_empty(&blkbk->pending_free)) {	140	if (!list_empty(&blkbk->pending_free)) {
144	req = list_entry(blkbk->pending_free.next, struct pending_req,	141	req = list_entry(blkbk->pending_free.next, struct pending_req,
145	free_list);	142	free_list);
146	list_del(&req->free_list);	143	list_del(&req->free_list);
147	}	144	}
148	spin_unlock_irqrestore(&blkbk->pending_free_lock, flags);	145	spin_unlock_irqrestore(&blkbk->pending_free_lock, flags);
149	return req;	146	return req;
150	}	147	}
151		148
152	/*	149	/*
153	* Return the 'pending_req' structure back to the freepool. We also	150	* Return the 'pending_req' structure back to the freepool. We also
154	* wake up the thread if it was waiting for a free page.	151	* wake up the thread if it was waiting for a free page.
155	*/	152	*/
156	static void free_req(struct pending_req *req)	153	static void free_req(struct pending_req *req)
157	{	154	{
158	unsigned long flags;	155	unsigned long flags;
159	int was_empty;	156	int was_empty;
160		157
161	spin_lock_irqsave(&blkbk->pending_free_lock, flags);	158	spin_lock_irqsave(&blkbk->pending_free_lock, flags);
162	was_empty = list_empty(&blkbk->pending_free);	159	was_empty = list_empty(&blkbk->pending_free);
163	list_add(&req->free_list, &blkbk->pending_free);	160	list_add(&req->free_list, &blkbk->pending_free);
164	spin_unlock_irqrestore(&blkbk->pending_free_lock, flags);	161	spin_unlock_irqrestore(&blkbk->pending_free_lock, flags);
165	if (was_empty)	162	if (was_empty)
166	wake_up(&blkbk->pending_free_wq);	163	wake_up(&blkbk->pending_free_wq);
167	}	164	}
168		165
169	/*	166	/*
170	* Routines for managing virtual block devices (vbds).	167	* Routines for managing virtual block devices (vbds).
171	*/	168	*/
172	static int xen_vbd_translate(struct phys_req req, struct xen_blkif blkif,	169	static int xen_vbd_translate(struct phys_req req, struct xen_blkif blkif,
173	int operation)	170	int operation)
174	{	171	{
175	struct xen_vbd *vbd = &blkif->vbd;	172	struct xen_vbd *vbd = &blkif->vbd;
176	int rc = -EACCES;	173	int rc = -EACCES;
177		174
178	if ((operation != READ) && vbd->readonly)	175	if ((operation != READ) && vbd->readonly)
179	goto out;	176	goto out;
180		177
181	if (likely(req->nr_sects)) {	178	if (likely(req->nr_sects)) {
182	blkif_sector_t end = req->sector_number + req->nr_sects;	179	blkif_sector_t end = req->sector_number + req->nr_sects;
183		180
184	if (unlikely(end < req->sector_number))	181	if (unlikely(end < req->sector_number))
185	goto out;	182	goto out;
186	if (unlikely(end > vbd_sz(vbd)))	183	if (unlikely(end > vbd_sz(vbd)))
187	goto out;	184	goto out;
188	}	185	}
189		186
190	req->dev = vbd->pdevice;	187	req->dev = vbd->pdevice;
191	req->bdev = vbd->bdev;	188	req->bdev = vbd->bdev;
192	rc = 0;	189	rc = 0;
193		190
194	out:	191	out:
195	return rc;	192	return rc;
196	}	193	}
197		194
198	static void xen_vbd_resize(struct xen_blkif *blkif)	195	static void xen_vbd_resize(struct xen_blkif *blkif)
199	{	196	{
200	struct xen_vbd *vbd = &blkif->vbd;	197	struct xen_vbd *vbd = &blkif->vbd;
201	struct xenbus_transaction xbt;	198	struct xenbus_transaction xbt;
202	int err;	199	int err;
203	struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be);	200	struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be);
204	unsigned long long new_size = vbd_sz(vbd);	201	unsigned long long new_size = vbd_sz(vbd);
205		202
206	pr_info(DRV_PFX "VBD Resize: Domid: %d, Device: (%d, %d)\n",	203	pr_info(DRV_PFX "VBD Resize: Domid: %d, Device: (%d, %d)\n",
207	blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice));	204	blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice));
208	pr_info(DRV_PFX "VBD Resize: new size %llu\n", new_size);	205	pr_info(DRV_PFX "VBD Resize: new size %llu\n", new_size);
209	vbd->size = new_size;	206	vbd->size = new_size;
210	again:	207	again:
211	err = xenbus_transaction_start(&xbt);	208	err = xenbus_transaction_start(&xbt);
212	if (err) {	209	if (err) {
213	pr_warn(DRV_PFX "Error starting transaction");	210	pr_warn(DRV_PFX "Error starting transaction");
214	return;	211	return;
215	}	212	}
216	err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu",	213	err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu",
217	(unsigned long long)vbd_sz(vbd));	214	(unsigned long long)vbd_sz(vbd));
218	if (err) {	215	if (err) {
219	pr_warn(DRV_PFX "Error writing new size");	216	pr_warn(DRV_PFX "Error writing new size");
220	goto abort;	217	goto abort;
221	}	218	}
222	/*	219	/*
223	* Write the current state; we will use this to synchronize	220	* Write the current state; we will use this to synchronize
224	* the front-end. If the current state is "connected" the	221	* the front-end. If the current state is "connected" the
225	* front-end will get the new size information online.	222	* front-end will get the new size information online.
226	*/	223	*/
227	err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state);	224	err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state);
228	if (err) {	225	if (err) {
229	pr_warn(DRV_PFX "Error writing the state");	226	pr_warn(DRV_PFX "Error writing the state");
230	goto abort;	227	goto abort;
231	}	228	}
232		229
233	err = xenbus_transaction_end(xbt, 0);	230	err = xenbus_transaction_end(xbt, 0);
234	if (err == -EAGAIN)	231	if (err == -EAGAIN)
235	goto again;	232	goto again;
236	if (err)	233	if (err)
237	pr_warn(DRV_PFX "Error ending transaction");	234	pr_warn(DRV_PFX "Error ending transaction");
238	return;	235	return;
239	abort:	236	abort:
240	xenbus_transaction_end(xbt, 1);	237	xenbus_transaction_end(xbt, 1);
241	}	238	}
242		239
243	/*	240	/*
244	* Notification from the guest OS.	241	* Notification from the guest OS.
245	*/	242	*/
246	static void blkif_notify_work(struct xen_blkif *blkif)	243	static void blkif_notify_work(struct xen_blkif *blkif)
247	{	244	{
248	blkif->waiting_reqs = 1;	245	blkif->waiting_reqs = 1;
249	wake_up(&blkif->wq);	246	wake_up(&blkif->wq);
250	}	247	}
251		248
252	irqreturn_t xen_blkif_be_int(int irq, void *dev_id)	249	irqreturn_t xen_blkif_be_int(int irq, void *dev_id)
253	{	250	{
254	blkif_notify_work(dev_id);	251	blkif_notify_work(dev_id);
255	return IRQ_HANDLED;	252	return IRQ_HANDLED;
256	}	253	}
257		254
258	/*	255	/*
259	* SCHEDULER FUNCTIONS	256	* SCHEDULER FUNCTIONS
260	*/	257	*/
261		258
262	static void print_stats(struct xen_blkif *blkif)	259	static void print_stats(struct xen_blkif *blkif)
263	{	260	{
264	pr_info("xen-blkback (%s): oo %3d \| rd %4d \| wr %4d \| f %4d"	261	pr_info("xen-blkback (%s): oo %3d \| rd %4d \| wr %4d \| f %4d"
265	" \| ds %4d\n",	262	" \| ds %4d\n",
266	current->comm, blkif->st_oo_req,	263	current->comm, blkif->st_oo_req,
267	blkif->st_rd_req, blkif->st_wr_req,	264	blkif->st_rd_req, blkif->st_wr_req,
268	blkif->st_f_req, blkif->st_ds_req);	265	blkif->st_f_req, blkif->st_ds_req);
269	blkif->st_print = jiffies + msecs_to_jiffies(10 * 1000);	266	blkif->st_print = jiffies + msecs_to_jiffies(10 * 1000);
270	blkif->st_rd_req = 0;	267	blkif->st_rd_req = 0;
271	blkif->st_wr_req = 0;	268	blkif->st_wr_req = 0;
272	blkif->st_oo_req = 0;	269	blkif->st_oo_req = 0;
273	blkif->st_ds_req = 0;	270	blkif->st_ds_req = 0;
274	}	271	}
275		272
276	int xen_blkif_schedule(void *arg)	273	int xen_blkif_schedule(void *arg)
277	{	274	{
278	struct xen_blkif *blkif = arg;	275	struct xen_blkif *blkif = arg;
279	struct xen_vbd *vbd = &blkif->vbd;	276	struct xen_vbd *vbd = &blkif->vbd;
280		277
281	xen_blkif_get(blkif);	278	xen_blkif_get(blkif);
282		279
283	while (!kthread_should_stop()) {	280	while (!kthread_should_stop()) {
284	if (try_to_freeze())	281	if (try_to_freeze())
285	continue;	282	continue;
286	if (unlikely(vbd->size != vbd_sz(vbd)))	283	if (unlikely(vbd->size != vbd_sz(vbd)))
287	xen_vbd_resize(blkif);	284	xen_vbd_resize(blkif);
288		285
289	wait_event_interruptible(	286	wait_event_interruptible(
290	blkif->wq,	287	blkif->wq,
291	blkif->waiting_reqs \|\| kthread_should_stop());	288	blkif->waiting_reqs \|\| kthread_should_stop());
292	wait_event_interruptible(	289	wait_event_interruptible(
293	blkbk->pending_free_wq,	290	blkbk->pending_free_wq,
294	!list_empty(&blkbk->pending_free) \|\|	291	!list_empty(&blkbk->pending_free) \|\|
295	kthread_should_stop());	292	kthread_should_stop());
296		293
297	blkif->waiting_reqs = 0;	294	blkif->waiting_reqs = 0;
298	smp_mb(); /* clear flag before checking for work */	295	smp_mb(); /* clear flag before checking for work */
299		296
300	if (do_block_io_op(blkif))	297	if (do_block_io_op(blkif))
301	blkif->waiting_reqs = 1;	298	blkif->waiting_reqs = 1;
302		299
303	if (log_stats && time_after(jiffies, blkif->st_print))	300	if (log_stats && time_after(jiffies, blkif->st_print))
304	print_stats(blkif);	301	print_stats(blkif);
305	}	302	}
306		303
307	if (log_stats)	304	if (log_stats)
308	print_stats(blkif);	305	print_stats(blkif);
309		306
310	blkif->xenblkd = NULL;	307	blkif->xenblkd = NULL;
311	xen_blkif_put(blkif);	308	xen_blkif_put(blkif);
312		309
313	return 0;	310	return 0;
314	}	311	}
315		312
316	struct seg_buf {	313	struct seg_buf {
317	unsigned long buf;	314	unsigned long buf;
318	unsigned int nsec;	315	unsigned int nsec;
319	};	316	};
320	/*	317	/*
321	* Unmap the grant references, and also remove the M2P over-rides	318	* Unmap the grant references, and also remove the M2P over-rides
322	* used in the 'pending_req'.	319	* used in the 'pending_req'.
323	*/	320	*/
324	static void xen_blkbk_unmap(struct pending_req *req)	321	static void xen_blkbk_unmap(struct pending_req *req)
325	{	322	{
326	struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];	323	struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
327	unsigned int i, invcount = 0;	324	unsigned int i, invcount = 0;
328	grant_handle_t handle;	325	grant_handle_t handle;
329	int ret;	326	int ret;
330		327
331	for (i = 0; i < req->nr_pages; i++) {	328	for (i = 0; i < req->nr_pages; i++) {
332	handle = pending_handle(req, i);	329	handle = pending_handle(req, i);
333	if (handle == BLKBACK_INVALID_HANDLE)	330	if (handle == BLKBACK_INVALID_HANDLE)
334	continue;	331	continue;
335	gnttab_set_unmap_op(&unmap[invcount], vaddr(req, i),	332	gnttab_set_unmap_op(&unmap[invcount], vaddr(req, i),
336	GNTMAP_host_map, handle);	333	GNTMAP_host_map, handle);
337	pending_handle(req, i) = BLKBACK_INVALID_HANDLE;	334	pending_handle(req, i) = BLKBACK_INVALID_HANDLE;
338	invcount++;	335	invcount++;
339	}	336	}
340		337
341	ret = HYPERVISOR_grant_table_op(	338	ret = HYPERVISOR_grant_table_op(
342	GNTTABOP_unmap_grant_ref, unmap, invcount);	339	GNTTABOP_unmap_grant_ref, unmap, invcount);
343	BUG_ON(ret);	340	BUG_ON(ret);
344	/*	341	/*
345	* Note, we use invcount, so nr->pages, so we can't index	342	* Note, we use invcount, so nr->pages, so we can't index
346	* using vaddr(req, i).	343	* using vaddr(req, i).
347	*/	344	*/
348	for (i = 0; i < invcount; i++) {	345	for (i = 0; i < invcount; i++) {
349	ret = m2p_remove_override(	346	ret = m2p_remove_override(
350	virt_to_page(unmap[i].host_addr), false);	347	virt_to_page(unmap[i].host_addr), false);
351	if (ret) {	348	if (ret) {
352	pr_alert(DRV_PFX "Failed to remove M2P override for %lx\n",	349	pr_alert(DRV_PFX "Failed to remove M2P override for %lx\n",
353	(unsigned long)unmap[i].host_addr);	350	(unsigned long)unmap[i].host_addr);
354	continue;	351	continue;
355	}	352	}
356	}	353	}
357	}	354	}
358		355
359	static int xen_blkbk_map(struct blkif_request *req,	356	static int xen_blkbk_map(struct blkif_request *req,
360	struct pending_req *pending_req,	357	struct pending_req *pending_req,
361	struct seg_buf seg[])	358	struct seg_buf seg[])
362	{	359	{
363	struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST];	360	struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST];
364	int i;	361	int i;
365	int nseg = req->nr_segments;	362	int nseg = req->u.rw.nr_segments;
366	int ret = 0;	363	int ret = 0;
367		364
368	/*	365	/*
369	* Fill out preq.nr_sects with proper amount of sectors, and setup	366	* Fill out preq.nr_sects with proper amount of sectors, and setup
370	* assign map[..] with the PFN of the page in our domain with the	367	* assign map[..] with the PFN of the page in our domain with the
371	* corresponding grant reference for each page.	368	* corresponding grant reference for each page.
372	*/	369	*/
373	for (i = 0; i < nseg; i++) {	370	for (i = 0; i < nseg; i++) {
374	uint32_t flags;	371	uint32_t flags;
375		372
376	flags = GNTMAP_host_map;	373	flags = GNTMAP_host_map;
377	if (pending_req->operation != BLKIF_OP_READ)	374	if (pending_req->operation != BLKIF_OP_READ)
378	flags \|= GNTMAP_readonly;	375	flags \|= GNTMAP_readonly;
379	gnttab_set_map_op(&map[i], vaddr(pending_req, i), flags,	376	gnttab_set_map_op(&map[i], vaddr(pending_req, i), flags,
380	req->u.rw.seg[i].gref,	377	req->u.rw.seg[i].gref,
381	pending_req->blkif->domid);	378	pending_req->blkif->domid);
382	}	379	}
383		380
384	ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map, nseg);	381	ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map, nseg);
385	BUG_ON(ret);	382	BUG_ON(ret);
386		383
387	/*	384	/*
388	* Now swizzle the MFN in our domain with the MFN from the other domain	385	* Now swizzle the MFN in our domain with the MFN from the other domain
389	* so that when we access vaddr(pending_req,i) it has the contents of	386	* so that when we access vaddr(pending_req,i) it has the contents of
390	* the page from the other domain.	387	* the page from the other domain.
391	*/	388	*/
392	for (i = 0; i < nseg; i++) {	389	for (i = 0; i < nseg; i++) {
393	if (unlikely(map[i].status != 0)) {	390	if (unlikely(map[i].status != 0)) {
394	pr_debug(DRV_PFX "invalid buffer -- could not remap it\n");	391	pr_debug(DRV_PFX "invalid buffer -- could not remap it\n");
395	map[i].handle = BLKBACK_INVALID_HANDLE;	392	map[i].handle = BLKBACK_INVALID_HANDLE;
396	ret \|= 1;	393	ret \|= 1;
397	}	394	}
398		395
399	pending_handle(pending_req, i) = map[i].handle;	396	pending_handle(pending_req, i) = map[i].handle;
400		397
401	if (ret)	398	if (ret)
402	continue;	399	continue;
403		400
404	ret = m2p_add_override(PFN_DOWN(map[i].dev_bus_addr),	401	ret = m2p_add_override(PFN_DOWN(map[i].dev_bus_addr),
405	blkbk->pending_page(pending_req, i), NULL);	402	blkbk->pending_page(pending_req, i), NULL);
406	if (ret) {	403	if (ret) {
407	pr_alert(DRV_PFX "Failed to install M2P override for %lx (ret: %d)\n",	404	pr_alert(DRV_PFX "Failed to install M2P override for %lx (ret: %d)\n",
408	(unsigned long)map[i].dev_bus_addr, ret);	405	(unsigned long)map[i].dev_bus_addr, ret);
409	/* We could switch over to GNTTABOP_copy */	406	/* We could switch over to GNTTABOP_copy */
410	continue;	407	continue;
411	}	408	}
412		409
413	seg[i].buf = map[i].dev_bus_addr \|	410	seg[i].buf = map[i].dev_bus_addr \|
414	(req->u.rw.seg[i].first_sect << 9);	411	(req->u.rw.seg[i].first_sect << 9);
415	}	412	}
416	return ret;	413	return ret;
417	}	414	}
418		415
419	static void xen_blk_discard(struct xen_blkif blkif, struct blkif_request req)	416	static int dispatch_discard_io(struct xen_blkif *blkif,
		417	struct blkif_request *req)
420	{	418	{
421	int err = 0;	419	int err = 0;
422	int status = BLKIF_RSP_OKAY;	420	int status = BLKIF_RSP_OKAY;
423	struct block_device *bdev = blkif->vbd.bdev;	421	struct block_device *bdev = blkif->vbd.bdev;
424		422
425	if (blkif->blk_backend_type == BLKIF_BACKEND_PHY)	423	blkif->st_ds_req++;
426	/* just forward the discard request */	424
		425	xen_blkif_get(blkif);
		426	if (blkif->blk_backend_type == BLKIF_BACKEND_PHY \|\|
		427	blkif->blk_backend_type == BLKIF_BACKEND_FILE) {
		428	unsigned long secure = (blkif->vbd.discard_secure &&
		429	(req->u.discard.flag & BLKIF_DISCARD_SECURE)) ?
		430	BLKDEV_DISCARD_SECURE : 0;
427	err = blkdev_issue_discard(bdev,	431	err = blkdev_issue_discard(bdev,
428	req->u.discard.sector_number,	432	req->u.discard.sector_number,
429	req->u.discard.nr_sectors,	433	req->u.discard.nr_sectors,
430	GFP_KERNEL, 0);	434	GFP_KERNEL, secure);
431	else if (blkif->blk_backend_type == BLKIF_BACKEND_FILE) {
432	/* punch a hole in the backing file */
433	struct loop_device *lo = bdev->bd_disk->private_data;
434	struct file *file = lo->lo_backing_file;
435
436	if (file->f_op->fallocate)
437	err = file->f_op->fallocate(file,
438	FALLOC_FL_KEEP_SIZE \| FALLOC_FL_PUNCH_HOLE,
439	req->u.discard.sector_number << 9,
440	req->u.discard.nr_sectors << 9);
441	else
442	err = -EOPNOTSUPP;
443	} else	435	} else
444	err = -EOPNOTSUPP;	436	err = -EOPNOTSUPP;
445		437
446	if (err == -EOPNOTSUPP) {	438	if (err == -EOPNOTSUPP) {
447	pr_debug(DRV_PFX "discard op failed, not supported\n");	439	pr_debug(DRV_PFX "discard op failed, not supported\n");
448	status = BLKIF_RSP_EOPNOTSUPP;	440	status = BLKIF_RSP_EOPNOTSUPP;
449	} else if (err)	441	} else if (err)
450	status = BLKIF_RSP_ERROR;	442	status = BLKIF_RSP_ERROR;
451		443
452	make_response(blkif, req->id, req->operation, status);	444	make_response(blkif, req->u.discard.id, req->operation, status);
		445	xen_blkif_put(blkif);
		446	return err;
453	}	447	}
454		448
455	static void xen_blk_drain_io(struct xen_blkif *blkif)	449	static void xen_blk_drain_io(struct xen_blkif *blkif)
456	{	450	{
457	atomic_set(&blkif->drain, 1);	451	atomic_set(&blkif->drain, 1);
458	do {	452	do {
459	/* The initial value is one, and one refcnt taken at the	453	/* The initial value is one, and one refcnt taken at the
460	* start of the xen_blkif_schedule thread. */	454	* start of the xen_blkif_schedule thread. */
461	if (atomic_read(&blkif->refcnt) <= 2)	455	if (atomic_read(&blkif->refcnt) <= 2)
462	break;	456	break;
463	wait_for_completion_interruptible_timeout(	457	wait_for_completion_interruptible_timeout(
464	&blkif->drain_complete, HZ);	458	&blkif->drain_complete, HZ);
465		459
466	if (!atomic_read(&blkif->drain))	460	if (!atomic_read(&blkif->drain))
467	break;	461	break;
468	} while (!kthread_should_stop());	462	} while (!kthread_should_stop());
469	atomic_set(&blkif->drain, 0);	463	atomic_set(&blkif->drain, 0);
470	}	464	}
471		465
472	/*	466	/*
473	* Completion callback on the bio's. Called as bh->b_end_io()	467	* Completion callback on the bio's. Called as bh->b_end_io()
474	*/	468	*/
475		469
476	static void __end_block_io_op(struct pending_req *pending_req, int error)	470	static void __end_block_io_op(struct pending_req *pending_req, int error)
477	{	471	{
478	/* An error fails the entire request. */	472	/* An error fails the entire request. */
479	if ((pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE) &&	473	if ((pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE) &&
480	(error == -EOPNOTSUPP)) {	474	(error == -EOPNOTSUPP)) {
481	pr_debug(DRV_PFX "flush diskcache op failed, not supported\n");	475	pr_debug(DRV_PFX "flush diskcache op failed, not supported\n");
482	xen_blkbk_flush_diskcache(XBT_NIL, pending_req->blkif->be, 0);	476	xen_blkbk_flush_diskcache(XBT_NIL, pending_req->blkif->be, 0);
483	pending_req->status = BLKIF_RSP_EOPNOTSUPP;	477	pending_req->status = BLKIF_RSP_EOPNOTSUPP;
484	} else if ((pending_req->operation == BLKIF_OP_WRITE_BARRIER) &&	478	} else if ((pending_req->operation == BLKIF_OP_WRITE_BARRIER) &&
485	(error == -EOPNOTSUPP)) {	479	(error == -EOPNOTSUPP)) {
486	pr_debug(DRV_PFX "write barrier op failed, not supported\n");	480	pr_debug(DRV_PFX "write barrier op failed, not supported\n");
487	xen_blkbk_barrier(XBT_NIL, pending_req->blkif->be, 0);	481	xen_blkbk_barrier(XBT_NIL, pending_req->blkif->be, 0);
488	pending_req->status = BLKIF_RSP_EOPNOTSUPP;	482	pending_req->status = BLKIF_RSP_EOPNOTSUPP;
489	} else if (error) {	483	} else if (error) {
490	pr_debug(DRV_PFX "Buffer not up-to-date at end of operation,"	484	pr_debug(DRV_PFX "Buffer not up-to-date at end of operation,"
491	" error=%d\n", error);	485	" error=%d\n", error);
492	pending_req->status = BLKIF_RSP_ERROR;	486	pending_req->status = BLKIF_RSP_ERROR;
493	}	487	}
494		488
495	/*	489	/*
496	* If all of the bio's have completed it is time to unmap	490	* If all of the bio's have completed it is time to unmap
497	* the grant references associated with 'request' and provide	491	* the grant references associated with 'request' and provide
498	* the proper response on the ring.	492	* the proper response on the ring.
499	*/	493	*/
500	if (atomic_dec_and_test(&pending_req->pendcnt)) {	494	if (atomic_dec_and_test(&pending_req->pendcnt)) {
501	xen_blkbk_unmap(pending_req);	495	xen_blkbk_unmap(pending_req);
502	make_response(pending_req->blkif, pending_req->id,	496	make_response(pending_req->blkif, pending_req->id,
503	pending_req->operation, pending_req->status);	497	pending_req->operation, pending_req->status);
504	xen_blkif_put(pending_req->blkif);	498	xen_blkif_put(pending_req->blkif);
505	if (atomic_read(&pending_req->blkif->refcnt) <= 2) {	499	if (atomic_read(&pending_req->blkif->refcnt) <= 2) {
506	if (atomic_read(&pending_req->blkif->drain))	500	if (atomic_read(&pending_req->blkif->drain))
507	complete(&pending_req->blkif->drain_complete);	501	complete(&pending_req->blkif->drain_complete);
508	}	502	}
509	free_req(pending_req);	503	free_req(pending_req);
510	}	504	}
511	}	505	}
512		506
513	/*	507	/*
514	* bio callback.	508	* bio callback.
515	*/	509	*/
516	static void end_block_io_op(struct bio *bio, int error)	510	static void end_block_io_op(struct bio *bio, int error)
517	{	511	{
518	__end_block_io_op(bio->bi_private, error);	512	__end_block_io_op(bio->bi_private, error);
519	bio_put(bio);	513	bio_put(bio);
520	}	514	}
521		515
522		516
523		517
524	/*	518	/*
525	* Function to copy the from the ring buffer the 'struct blkif_request'	519	* Function to copy the from the ring buffer the 'struct blkif_request'
526	* (which has the sectors we want, number of them, grant references, etc),	520	* (which has the sectors we want, number of them, grant references, etc),
527	* and transmute it to the block API to hand it over to the proper block disk.	521	* and transmute it to the block API to hand it over to the proper block disk.
528	*/	522	*/
529	static int	523	static int
530	__do_block_io_op(struct xen_blkif *blkif)	524	__do_block_io_op(struct xen_blkif *blkif)
531	{	525	{
532	union blkif_back_rings *blk_rings = &blkif->blk_rings;	526	union blkif_back_rings *blk_rings = &blkif->blk_rings;
533	struct blkif_request req;	527	struct blkif_request req;
534	struct pending_req *pending_req;	528	struct pending_req *pending_req;
535	RING_IDX rc, rp;	529	RING_IDX rc, rp;
536	int more_to_do = 0;	530	int more_to_do = 0;
537		531
538	rc = blk_rings->common.req_cons;	532	rc = blk_rings->common.req_cons;
539	rp = blk_rings->common.sring->req_prod;	533	rp = blk_rings->common.sring->req_prod;
540	rmb(); /* Ensure we see queued requests up to 'rp'. */	534	rmb(); /* Ensure we see queued requests up to 'rp'. */
541		535
542	while (rc != rp) {	536	while (rc != rp) {
543		537
544	if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc))	538	if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc))
545	break;	539	break;
546		540
547	if (kthread_should_stop()) {	541	if (kthread_should_stop()) {
548	more_to_do = 1;	542	more_to_do = 1;
549	break;	543	break;
550	}	544	}
551		545
552	pending_req = alloc_req();	546	pending_req = alloc_req();
553	if (NULL == pending_req) {	547	if (NULL == pending_req) {
554	blkif->st_oo_req++;	548	blkif->st_oo_req++;
555	more_to_do = 1;	549	more_to_do = 1;
556	break;	550	break;
557	}	551	}
558		552
559	switch (blkif->blk_protocol) {	553	switch (blkif->blk_protocol) {
560	case BLKIF_PROTOCOL_NATIVE:	554	case BLKIF_PROTOCOL_NATIVE:
561	memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req));	555	memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req));
562	break;	556	break;
563	case BLKIF_PROTOCOL_X86_32:	557	case BLKIF_PROTOCOL_X86_32:
564	blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc));	558	blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc));
565	break;	559	break;
566	case BLKIF_PROTOCOL_X86_64:	560	case BLKIF_PROTOCOL_X86_64:
567	blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc));	561	blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc));
568	break;	562	break;
569	default:	563	default:
570	BUG();	564	BUG();
571	}	565	}
572	blk_rings->common.req_cons = ++rc; /* before make_response() */	566	blk_rings->common.req_cons = ++rc; /* before make_response() */
573		567
574	/* Apply all sanity checks to /private copy/ of request. */	568	/* Apply all sanity checks to /private copy/ of request. */
575	barrier();	569	barrier();
576		570	if (unlikely(req.operation == BLKIF_OP_DISCARD)) {
577	if (dispatch_rw_block_io(blkif, &req, pending_req))	571	free_req(pending_req);
		572	if (dispatch_discard_io(blkif, &req))
		573	break;
		574	} else if (dispatch_rw_block_io(blkif, &req, pending_req))
578	break;	575	break;
579		576
580	/* Yield point for this unbounded loop. */	577	/* Yield point for this unbounded loop. */
581	cond_resched();	578	cond_resched();
582	}	579	}
583		580
584	return more_to_do;	581	return more_to_do;
585	}	582	}
586		583
587	static int	584	static int
588	do_block_io_op(struct xen_blkif *blkif)	585	do_block_io_op(struct xen_blkif *blkif)
589	{	586	{
590	union blkif_back_rings *blk_rings = &blkif->blk_rings;	587	union blkif_back_rings *blk_rings = &blkif->blk_rings;
591	int more_to_do;	588	int more_to_do;
592		589
593	do {	590	do {
594	more_to_do = __do_block_io_op(blkif);	591	more_to_do = __do_block_io_op(blkif);
595	if (more_to_do)	592	if (more_to_do)
596	break;	593	break;
597		594
598	RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do);	595	RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do);
599	} while (more_to_do);	596	} while (more_to_do);
600		597
601	return more_to_do;	598	return more_to_do;
602	}	599	}
603	/*	600	/*
604	* Transmutation of the 'struct blkif_request' to a proper 'struct bio'	601	* Transmutation of the 'struct blkif_request' to a proper 'struct bio'
605	* and call the 'submit_bio' to pass it to the underlying storage.	602	* and call the 'submit_bio' to pass it to the underlying storage.
606	*/	603	*/
607	static int dispatch_rw_block_io(struct xen_blkif *blkif,	604	static int dispatch_rw_block_io(struct xen_blkif *blkif,
608	struct blkif_request *req,	605	struct blkif_request *req,
609	struct pending_req *pending_req)	606	struct pending_req *pending_req)
610	{	607	{
611	struct phys_req preq;	608	struct phys_req preq;
612	struct seg_buf seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];	609	struct seg_buf seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
613	unsigned int nseg;	610	unsigned int nseg;
614	struct bio *bio = NULL;	611	struct bio *bio = NULL;
615	struct bio *biolist[BLKIF_MAX_SEGMENTS_PER_REQUEST];	612	struct bio *biolist[BLKIF_MAX_SEGMENTS_PER_REQUEST];
616	int i, nbio = 0;	613	int i, nbio = 0;
617	int operation;	614	int operation;
618	struct blk_plug plug;	615	struct blk_plug plug;
619	bool drain = false;	616	bool drain = false;
620		617
621	switch (req->operation) {	618	switch (req->operation) {
622	case BLKIF_OP_READ:	619	case BLKIF_OP_READ:
623	blkif->st_rd_req++;	620	blkif->st_rd_req++;
624	operation = READ;	621	operation = READ;
625	break;	622	break;
626	case BLKIF_OP_WRITE:	623	case BLKIF_OP_WRITE:
627	blkif->st_wr_req++;	624	blkif->st_wr_req++;
628	operation = WRITE_ODIRECT;	625	operation = WRITE_ODIRECT;
629	break;	626	break;
630	case BLKIF_OP_WRITE_BARRIER:	627	case BLKIF_OP_WRITE_BARRIER:
631	drain = true;	628	drain = true;
632	case BLKIF_OP_FLUSH_DISKCACHE:	629	case BLKIF_OP_FLUSH_DISKCACHE:
633	blkif->st_f_req++;	630	blkif->st_f_req++;
634	operation = WRITE_FLUSH;	631	operation = WRITE_FLUSH;
635	break;	632	break;
636	case BLKIF_OP_DISCARD:
637	blkif->st_ds_req++;
638	operation = REQ_DISCARD;
639	break;
640	default:	633	default:
641	operation = 0; /* make gcc happy */	634	operation = 0; /* make gcc happy */
642	goto fail_response;	635	goto fail_response;
643	break;	636	break;
644	}	637	}
645		638
646	/* Check that the number of segments is sane. */	639	/* Check that the number of segments is sane. */
647	nseg = req->nr_segments;	640	nseg = req->u.rw.nr_segments;
648	if (unlikely(nseg == 0 && operation != WRITE_FLUSH &&	641
649	operation != REQ_DISCARD) \|\|	642	if (unlikely(nseg == 0 && operation != WRITE_FLUSH) \|\|
650	unlikely(nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) {	643	unlikely(nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
651	pr_debug(DRV_PFX "Bad number of segments in request (%d)\n",	644	pr_debug(DRV_PFX "Bad number of segments in request (%d)\n",
652	nseg);	645	nseg);
653	/* Haven't submitted any bio's yet. */	646	/* Haven't submitted any bio's yet. */
654	goto fail_response;	647	goto fail_response;
655	}	648	}
656		649
657	preq.dev = req->handle;	650	preq.dev = req->u.rw.handle;
658	preq.sector_number = req->u.rw.sector_number;	651	preq.sector_number = req->u.rw.sector_number;
659	preq.nr_sects = 0;	652	preq.nr_sects = 0;
660		653
661	pending_req->blkif = blkif;	654	pending_req->blkif = blkif;
662	pending_req->id = req->id;	655	pending_req->id = req->u.rw.id;
663	pending_req->operation = req->operation;	656	pending_req->operation = req->operation;
664	pending_req->status = BLKIF_RSP_OKAY;	657	pending_req->status = BLKIF_RSP_OKAY;
665	pending_req->nr_pages = nseg;	658	pending_req->nr_pages = nseg;
666		659
667	for (i = 0; i < nseg; i++) {	660	for (i = 0; i < nseg; i++) {
668	seg[i].nsec = req->u.rw.seg[i].last_sect -	661	seg[i].nsec = req->u.rw.seg[i].last_sect -
669	req->u.rw.seg[i].first_sect + 1;	662	req->u.rw.seg[i].first_sect + 1;
670	if ((req->u.rw.seg[i].last_sect >= (PAGE_SIZE >> 9)) \|\|	663	if ((req->u.rw.seg[i].last_sect >= (PAGE_SIZE >> 9)) \|\|
671	(req->u.rw.seg[i].last_sect < req->u.rw.seg[i].first_sect))	664	(req->u.rw.seg[i].last_sect < req->u.rw.seg[i].first_sect))
672	goto fail_response;	665	goto fail_response;
673	preq.nr_sects += seg[i].nsec;	666	preq.nr_sects += seg[i].nsec;
674		667
675	}	668	}
676		669
677	if (xen_vbd_translate(&preq, blkif, operation) != 0) {	670	if (xen_vbd_translate(&preq, blkif, operation) != 0) {
678	pr_debug(DRV_PFX "access denied: %s of [%llu,%llu] on dev=%04x\n",	671	pr_debug(DRV_PFX "access denied: %s of [%llu,%llu] on dev=%04x\n",
679	operation == READ ? "read" : "write",	672	operation == READ ? "read" : "write",
680	preq.sector_number,	673	preq.sector_number,
681	preq.sector_number + preq.nr_sects, preq.dev);	674	preq.sector_number + preq.nr_sects, preq.dev);
682	goto fail_response;	675	goto fail_response;
683	}	676	}
684		677
685	/*	678	/*
686	* This check _MUST_ be done after xen_vbd_translate as the preq.bdev	679	* This check _MUST_ be done after xen_vbd_translate as the preq.bdev
687	* is set there.	680	* is set there.
688	*/	681	*/
689	for (i = 0; i < nseg; i++) {	682	for (i = 0; i < nseg; i++) {
690	if (((int)preq.sector_number\|(int)seg[i].nsec) &	683	if (((int)preq.sector_number\|(int)seg[i].nsec) &
691	((bdev_logical_block_size(preq.bdev) >> 9) - 1)) {	684	((bdev_logical_block_size(preq.bdev) >> 9) - 1)) {
692	pr_debug(DRV_PFX "Misaligned I/O request from domain %d",	685	pr_debug(DRV_PFX "Misaligned I/O request from domain %d",
693	blkif->domid);	686	blkif->domid);
694	goto fail_response;	687	goto fail_response;
695	}	688	}
696	}	689	}
697		690
698	/* Wait on all outstanding I/O's and once that has been completed	691	/* Wait on all outstanding I/O's and once that has been completed
699	* issue the WRITE_FLUSH.	692	* issue the WRITE_FLUSH.
700	*/	693	*/
701	if (drain)	694	if (drain)
702	xen_blk_drain_io(pending_req->blkif);	695	xen_blk_drain_io(pending_req->blkif);
703		696
704	/*	697	/*
705	* If we have failed at this point, we need to undo the M2P override,	698	* If we have failed at this point, we need to undo the M2P override,
706	* set gnttab_set_unmap_op on all of the grant references and perform	699	* set gnttab_set_unmap_op on all of the grant references and perform
707	* the hypercall to unmap the grants - that is all done in	700	* the hypercall to unmap the grants - that is all done in
708	* xen_blkbk_unmap.	701	* xen_blkbk_unmap.
709	*/	702	*/
710	if (operation != REQ_DISCARD && xen_blkbk_map(req, pending_req, seg))	703	if (xen_blkbk_map(req, pending_req, seg))
711	goto fail_flush;	704	goto fail_flush;
712		705
713	/*	706	/*
714	* This corresponding xen_blkif_put is done in __end_block_io_op, or	707	* This corresponding xen_blkif_put is done in __end_block_io_op, or
715	* below (in "!bio") if we are handling a BLKIF_OP_DISCARD.	708	* below (in "!bio") if we are handling a BLKIF_OP_DISCARD.
716	*/	709	*/
717	xen_blkif_get(blkif);	710	xen_blkif_get(blkif);
718		711
719	for (i = 0; i < nseg; i++) {	712	for (i = 0; i < nseg; i++) {
720	while ((bio == NULL) \|\|	713	while ((bio == NULL) \|\|
721	(bio_add_page(bio,	714	(bio_add_page(bio,
722	blkbk->pending_page(pending_req, i),	715	blkbk->pending_page(pending_req, i),
723	seg[i].nsec << 9,	716	seg[i].nsec << 9,
724	seg[i].buf & ~PAGE_MASK) == 0)) {	717	seg[i].buf & ~PAGE_MASK) == 0)) {
725		718
726	bio = bio_alloc(GFP_KERNEL, nseg-i);	719	bio = bio_alloc(GFP_KERNEL, nseg-i);
727	if (unlikely(bio == NULL))	720	if (unlikely(bio == NULL))
728	goto fail_put_bio;	721	goto fail_put_bio;
729		722
730	biolist[nbio++] = bio;	723	biolist[nbio++] = bio;
731	bio->bi_bdev = preq.bdev;	724	bio->bi_bdev = preq.bdev;
732	bio->bi_private = pending_req;	725	bio->bi_private = pending_req;
733	bio->bi_end_io = end_block_io_op;	726	bio->bi_end_io = end_block_io_op;
734	bio->bi_sector = preq.sector_number;	727	bio->bi_sector = preq.sector_number;
735	}	728	}
736		729
737	preq.sector_number += seg[i].nsec;	730	preq.sector_number += seg[i].nsec;
738	}	731	}
739		732
740	/* This will be hit if the operation was a flush or discard. */	733	/* This will be hit if the operation was a flush or discard. */
741	if (!bio) {	734	if (!bio) {
742	BUG_ON(operation != WRITE_FLUSH && operation != REQ_DISCARD);	735	BUG_ON(operation != WRITE_FLUSH);
743		736
744	if (operation == WRITE_FLUSH) {	737	bio = bio_alloc(GFP_KERNEL, 0);
745	bio = bio_alloc(GFP_KERNEL, 0);	738	if (unlikely(bio == NULL))
746	if (unlikely(bio == NULL))	739	goto fail_put_bio;
747	goto fail_put_bio;
748		740
749	biolist[nbio++] = bio;	741	biolist[nbio++] = bio;
750	bio->bi_bdev = preq.bdev;	742	bio->bi_bdev = preq.bdev;
751	bio->bi_private = pending_req;	743	bio->bi_private = pending_req;
752	bio->bi_end_io = end_block_io_op;	744	bio->bi_end_io = end_block_io_op;
753	} else if (operation == REQ_DISCARD) {
754	xen_blk_discard(blkif, req);
755	xen_blkif_put(blkif);
756	free_req(pending_req);
757	return 0;
758	}
759	}	745	}
760		746
761	/*	747	/*
762	* We set it one so that the last submit_bio does not have to call	748	* We set it one so that the last submit_bio does not have to call
763	* atomic_inc.	749	* atomic_inc.
764	*/	750	*/
765	atomic_set(&pending_req->pendcnt, nbio);	751	atomic_set(&pending_req->pendcnt, nbio);
766		752
767	/* Get a reference count for the disk queue and start sending I/O */	753	/* Get a reference count for the disk queue and start sending I/O */
768	blk_start_plug(&plug);	754	blk_start_plug(&plug);
769		755
770	for (i = 0; i < nbio; i++)	756	for (i = 0; i < nbio; i++)
771	submit_bio(operation, biolist[i]);	757	submit_bio(operation, biolist[i]);
772		758
773	/* Let the I/Os go.. */	759	/* Let the I/Os go.. */
774	blk_finish_plug(&plug);	760	blk_finish_plug(&plug);
775		761
776	if (operation == READ)	762	if (operation == READ)
777	blkif->st_rd_sect += preq.nr_sects;	763	blkif->st_rd_sect += preq.nr_sects;
778	else if (operation & WRITE)	764	else if (operation & WRITE)
779	blkif->st_wr_sect += preq.nr_sects;	765	blkif->st_wr_sect += preq.nr_sects;
780		766
781	return 0;	767	return 0;
782		768
783	fail_flush:	769	fail_flush:
784	xen_blkbk_unmap(pending_req);	770	xen_blkbk_unmap(pending_req);
785	fail_response:	771	fail_response:
786	/* Haven't submitted any bio's yet. */	772	/* Haven't submitted any bio's yet. */
787	make_response(blkif, req->id, req->operation, BLKIF_RSP_ERROR);	773	make_response(blkif, req->u.rw.id, req->operation, BLKIF_RSP_ERROR);
788	free_req(pending_req);	774	free_req(pending_req);
789	msleep(1); /* back off a bit */	775	msleep(1); /* back off a bit */
790	return -EIO;	776	return -EIO;
791		777
792	fail_put_bio:	778	fail_put_bio:
793	for (i = 0; i < nbio; i++)	779	for (i = 0; i < nbio; i++)
794	bio_put(biolist[i]);	780	bio_put(biolist[i]);
795	__end_block_io_op(pending_req, -EINVAL);	781	__end_block_io_op(pending_req, -EINVAL);
796	msleep(1); /* back off a bit */	782	msleep(1); /* back off a bit */
797	return -EIO;	783	return -EIO;
798	}	784	}
799		785
800		786
801		787
802	/*	788	/*
803	* Put a response on the ring on how the operation fared.	789	* Put a response on the ring on how the operation fared.
804	*/	790	*/
805	static void make_response(struct xen_blkif *blkif, u64 id,	791	static void make_response(struct xen_blkif *blkif, u64 id,
806	unsigned short op, int st)	792	unsigned short op, int st)
807	{	793	{
808	struct blkif_response resp;	794	struct blkif_response resp;
809	unsigned long flags;	795	unsigned long flags;
810	union blkif_back_rings *blk_rings = &blkif->blk_rings;	796	union blkif_back_rings *blk_rings = &blkif->blk_rings;
811	int notify;	797	int notify;
812		798
813	resp.id = id;	799	resp.id = id;
814	resp.operation = op;	800	resp.operation = op;
815	resp.status = st;	801	resp.status = st;
816		802
817	spin_lock_irqsave(&blkif->blk_ring_lock, flags);	803	spin_lock_irqsave(&blkif->blk_ring_lock, flags);
818	/* Place on the response ring for the relevant domain. */	804	/* Place on the response ring for the relevant domain. */
819	switch (blkif->blk_protocol) {	805	switch (blkif->blk_protocol) {
820	case BLKIF_PROTOCOL_NATIVE:	806	case BLKIF_PROTOCOL_NATIVE:
821	memcpy(RING_GET_RESPONSE(&blk_rings->native, blk_rings->native.rsp_prod_pvt),	807	memcpy(RING_GET_RESPONSE(&blk_rings->native, blk_rings->native.rsp_prod_pvt),
822	&resp, sizeof(resp));	808	&resp, sizeof(resp));
823	break;	809	break;
824	case BLKIF_PROTOCOL_X86_32:	810	case BLKIF_PROTOCOL_X86_32:
825	memcpy(RING_GET_RESPONSE(&blk_rings->x86_32, blk_rings->x86_32.rsp_prod_pvt),	811	memcpy(RING_GET_RESPONSE(&blk_rings->x86_32, blk_rings->x86_32.rsp_prod_pvt),
826	&resp, sizeof(resp));	812	&resp, sizeof(resp));
827	break;	813	break;
828	case BLKIF_PROTOCOL_X86_64:	814	case BLKIF_PROTOCOL_X86_64:
829	memcpy(RING_GET_RESPONSE(&blk_rings->x86_64, blk_rings->x86_64.rsp_prod_pvt),	815	memcpy(RING_GET_RESPONSE(&blk_rings->x86_64, blk_rings->x86_64.rsp_prod_pvt),
830	&resp, sizeof(resp));	816	&resp, sizeof(resp));
831	break;	817	break;
832	default:	818	default:
833	BUG();	819	BUG();
834	}	820	}
835	blk_rings->common.rsp_prod_pvt++;	821	blk_rings->common.rsp_prod_pvt++;
836	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify);	822	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify);
837	spin_unlock_irqrestore(&blkif->blk_ring_lock, flags);	823	spin_unlock_irqrestore(&blkif->blk_ring_lock, flags);
838	if (notify)	824	if (notify)
839	notify_remote_via_irq(blkif->irq);	825	notify_remote_via_irq(blkif->irq);
840	}	826	}
841		827
842	static int __init xen_blkif_init(void)	828	static int __init xen_blkif_init(void)
843	{	829	{
844	int i, mmap_pages;	830	int i, mmap_pages;
845	int rc = 0;	831	int rc = 0;
846		832
847	if (!xen_pv_domain())	833	if (!xen_pv_domain())
848	return -ENODEV;	834	return -ENODEV;
849		835
850	blkbk = kzalloc(sizeof(struct xen_blkbk), GFP_KERNEL);	836	blkbk = kzalloc(sizeof(struct xen_blkbk), GFP_KERNEL);
851	if (!blkbk) {	837	if (!blkbk) {
852	pr_alert(DRV_PFX "%s: out of memory!\n", __func__);	838	pr_alert(DRV_PFX "%s: out of memory!\n", __func__);
853	return -ENOMEM;	839	return -ENOMEM;
854	}	840	}
855		841
856	mmap_pages = xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST;	842	mmap_pages = xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST;
857		843
858	blkbk->pending_reqs = kzalloc(sizeof(blkbk->pending_reqs[0]) *	844	blkbk->pending_reqs = kzalloc(sizeof(blkbk->pending_reqs[0]) *
859	xen_blkif_reqs, GFP_KERNEL);	845	xen_blkif_reqs, GFP_KERNEL);
860	blkbk->pending_grant_handles = kmalloc(sizeof(blkbk->pending_grant_handles[0]) *	846	blkbk->pending_grant_handles = kmalloc(sizeof(blkbk->pending_grant_handles[0]) *
861	mmap_pages, GFP_KERNEL);	847	mmap_pages, GFP_KERNEL);
862	blkbk->pending_pages = kzalloc(sizeof(blkbk->pending_pages[0]) *	848	blkbk->pending_pages = kzalloc(sizeof(blkbk->pending_pages[0]) *
863	mmap_pages, GFP_KERNEL);	849	mmap_pages, GFP_KERNEL);
864		850
865	if (!blkbk->pending_reqs \|\| !blkbk->pending_grant_handles \|\|	851	if (!blkbk->pending_reqs \|\| !blkbk->pending_grant_handles \|\|
866	!blkbk->pending_pages) {	852	!blkbk->pending_pages) {
867	rc = -ENOMEM;	853	rc = -ENOMEM;
868	goto out_of_memory;	854	goto out_of_memory;
869	}	855	}
870		856
871	for (i = 0; i < mmap_pages; i++) {	857	for (i = 0; i < mmap_pages; i++) {
872	blkbk->pending_grant_handles[i] = BLKBACK_INVALID_HANDLE;	858	blkbk->pending_grant_handles[i] = BLKBACK_INVALID_HANDLE;
873	blkbk->pending_pages[i] = alloc_page(GFP_KERNEL);	859	blkbk->pending_pages[i] = alloc_page(GFP_KERNEL);
874	if (blkbk->pending_pages[i] == NULL) {	860	if (blkbk->pending_pages[i] == NULL) {
875	rc = -ENOMEM;	861	rc = -ENOMEM;
876	goto out_of_memory;	862	goto out_of_memory;
877	}	863	}
878	}	864	}
879	rc = xen_blkif_interface_init();	865	rc = xen_blkif_interface_init();
880	if (rc)	866	if (rc)
881	goto failed_init;	867	goto failed_init;
882		868
883	INIT_LIST_HEAD(&blkbk->pending_free);	869	INIT_LIST_HEAD(&blkbk->pending_free);
884	spin_lock_init(&blkbk->pending_free_lock);	870	spin_lock_init(&blkbk->pending_free_lock);
885	init_waitqueue_head(&blkbk->pending_free_wq);	871	init_waitqueue_head(&blkbk->pending_free_wq);
886		872
887	for (i = 0; i < xen_blkif_reqs; i++)	873	for (i = 0; i < xen_blkif_reqs; i++)
888	list_add_tail(&blkbk->pending_reqs[i].free_list,	874	list_add_tail(&blkbk->pending_reqs[i].free_list,
889	&blkbk->pending_free);	875	&blkbk->pending_free);
890		876
891	rc = xen_blkif_xenbus_init();	877	rc = xen_blkif_xenbus_init();
892	if (rc)	878	if (rc)
893	goto failed_init;	879	goto failed_init;
894		880
895	return 0;	881	return 0;
896		882
897	out_of_memory:	883	out_of_memory:
898	pr_alert(DRV_PFX "%s: out of memory\n", __func__);	884	pr_alert(DRV_PFX "%s: out of memory\n", __func__);
899	failed_init:	885	failed_init:
900	kfree(blkbk->pending_reqs);	886	kfree(blkbk->pending_reqs);
901	kfree(blkbk->pending_grant_handles);	887	kfree(blkbk->pending_grant_handles);
902	if (blkbk->pending_pages) {	888	if (blkbk->pending_pages) {
903	for (i = 0; i < mmap_pages; i++) {	889	for (i = 0; i < mmap_pages; i++) {
904	if (blkbk->pending_pages[i])	890	if (blkbk->pending_pages[i])
905	__free_page(blkbk->pending_pages[i]);	891	__free_page(blkbk->pending_pages[i]);
906	}	892	}

drivers/block/xen-blkback/common.h

Diff comments View file @ 16008d6

 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License version 2
  * as published by the Free Software Foundation; or, when distributed
  * separately from the Linux kernel or incorporated into other
  * software packages, subject to the following license:
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this source file (the "Software"), to deal in the Software without
  * restriction, including without limitation the rights to use, copy, modify,
  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  * and to permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in
  * all copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  * IN THE SOFTWARE.
  */
 #ifndef __XEN_BLKIF__BACKEND__COMMON_H__
 #define __XEN_BLKIF__BACKEND__COMMON_H__
 #include <linux/module.h>
 #include <linux/interrupt.h>
 #include <linux/slab.h>
 #include <linux/blkdev.h>
 #include <linux/vmalloc.h>
 #include <linux/wait.h>
 #include <linux/io.h>
 #include <asm/setup.h>
 #include <asm/pgalloc.h>
 #include <asm/hypervisor.h>
 #include <xen/grant_table.h>
 #include <xen/xenbus.h>
 #include <xen/interface/io/ring.h>
 #include <xen/interface/io/blkif.h>
 #include <xen/interface/io/protocols.h>
 #define DRV_PFX "xen-blkback:"
 #define DPRINTK(fmt, args...)				\
 	pr_debug(DRV_PFX "(%s:%d) " fmt ".\n",		\
 		 __func__, __LINE__, ##args)
 /* Not a real protocol.  Used to generate ring structs which contain
  * the elements common to all protocols only.  This way we get a
  * compiler-checkable way to use common struct elements, so we can
  * avoid using switch(protocol) in a number of places.  */
 struct blkif_common_request {
 	char dummy;
 };
 struct blkif_common_response {
 	char dummy;
 };
-/* i386 protocol version */
-#pragma pack(push, 4)
 struct blkif_x86_32_request_rw {
+	uint8_t        nr_segments;  /* number of segments                   */
+	blkif_vdev_t   handle;       /* only for read/write requests         */
+	uint64_t       id;           /* private guest value, echoed in resp  */
 	blkif_sector_t sector_number;/* start sector idx on disk (r/w only)  */
 	struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
-};
+} __attribute__((__packed__));
 struct blkif_x86_32_request_discard {
+	uint8_t        flag;         /* BLKIF_DISCARD_SECURE or zero         */
+	blkif_vdev_t   _pad1;        /* was "handle" for read/write requests */
+	uint64_t       id;           /* private guest value, echoed in resp  */
 	blkif_sector_t sector_number;/* start sector idx on disk (r/w only)  */
-	uint64_t nr_sectors;
+	uint64_t       nr_sectors;
-};
+} __attribute__((__packed__));
 struct blkif_x86_32_request {
 	uint8_t        operation;    /* BLKIF_OP_???                         */
-	uint8_t        nr_segments;  /* number of segments                   */
-	blkif_vdev_t   handle;       /* only for read/write requests         */
-	uint64_t       id;           /* private guest value, echoed in resp  */
 	union {
 		struct blkif_x86_32_request_rw rw;
 		struct blkif_x86_32_request_discard discard;
 	} u;
-};
+} __attribute__((__packed__));
+/* i386 protocol version */
+#pragma pack(push, 4)
 struct blkif_x86_32_response {
 	uint64_t        id;              /* copied from request */
 	uint8_t         operation;       /* copied from request */
 	int16_t         status;          /* BLKIF_RSP_???       */
 };
 #pragma pack(pop)
 /* x86_64 protocol version */
 struct blkif_x86_64_request_rw {
+	uint8_t        nr_segments;  /* number of segments                   */
+	blkif_vdev_t   handle;       /* only for read/write requests         */
+	uint32_t       _pad1;        /* offsetof(blkif_reqest..,u.rw.id)==8  */
+	uint64_t       id;
 	blkif_sector_t sector_number;/* start sector idx on disk (r/w only)  */
 	struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
-};
+} __attribute__((__packed__));
 struct blkif_x86_64_request_discard {
+	uint8_t        flag;         /* BLKIF_DISCARD_SECURE or zero         */
+	blkif_vdev_t   _pad1;        /* was "handle" for read/write requests */
+        uint32_t       _pad2;        /* offsetof(blkif_..,u.discard.id)==8   */
+	uint64_t       id;
 	blkif_sector_t sector_number;/* start sector idx on disk (r/w only)  */
-	uint64_t nr_sectors;
+	uint64_t       nr_sectors;
-};
+} __attribute__((__packed__));
 struct blkif_x86_64_request {
 	uint8_t        operation;    /* BLKIF_OP_???                         */
-	uint8_t        nr_segments;  /* number of segments                   */
-	blkif_vdev_t   handle;       /* only for read/write requests         */
-	uint64_t       __attribute__((__aligned__(8))) id;
 	union {
 		struct blkif_x86_64_request_rw rw;
 		struct blkif_x86_64_request_discard discard;
 	} u;
-};
+} __attribute__((__packed__));
 struct blkif_x86_64_response {
 	uint64_t       __attribute__((__aligned__(8))) id;
 	uint8_t         operation;       /* copied from request */
 	int16_t         status;          /* BLKIF_RSP_???       */
 };
 DEFINE_RING_TYPES(blkif_common, struct blkif_common_request,
 		  struct blkif_common_response);
 DEFINE_RING_TYPES(blkif_x86_32, struct blkif_x86_32_request,
 		  struct blkif_x86_32_response);
 DEFINE_RING_TYPES(blkif_x86_64, struct blkif_x86_64_request,
 		  struct blkif_x86_64_response);
 union blkif_back_rings {
 	struct blkif_back_ring        native;
 	struct blkif_common_back_ring common;
 	struct blkif_x86_32_back_ring x86_32;
 	struct blkif_x86_64_back_ring x86_64;
 };
 enum blkif_protocol {
 	BLKIF_PROTOCOL_NATIVE = 1,
 	BLKIF_PROTOCOL_X86_32 = 2,
 	BLKIF_PROTOCOL_X86_64 = 3,
 };
 enum blkif_backend_type {
 	BLKIF_BACKEND_PHY  = 1,
 	BLKIF_BACKEND_FILE = 2,
 };
 struct xen_vbd {
 	/* What the domain refers to this vbd as. */
 	blkif_vdev_t		handle;
 	/* Non-zero -> read-only */
 	unsigned char		readonly;
 	/* VDISK_xxx */
 	unsigned char		type;
 	/* phys device that this vbd maps to. */
 	u32			pdevice;
 	struct block_device	*bdev;
 	/* Cached size parameter. */
 	sector_t		size;
 	bool			flush_support;
+	bool			discard_secure;
 };
 struct backend_info;
 struct xen_blkif {
 	/* Unique identifier for this interface. */
 	domid_t			domid;
 	unsigned int		handle;
 	/* Physical parameters of the comms window. */
 	unsigned int		irq;
 	/* Comms information. */
 	enum blkif_protocol	blk_protocol;
 	enum blkif_backend_type blk_backend_type;
 	union blkif_back_rings	blk_rings;
 	void			*blk_ring;
 	/* The VBD attached to this interface. */
 	struct xen_vbd		vbd;
 	/* Back pointer to the backend_info. */
 	struct backend_info	*be;
 	/* Private fields. */
 	spinlock_t		blk_ring_lock;
 	atomic_t		refcnt;
 	wait_queue_head_t	wq;
 	/* for barrier (drain) requests */
 	struct completion	drain_complete;
 	atomic_t		drain;
 	/* One thread per one blkif. */
 	struct task_struct	*xenblkd;
 	unsigned int		waiting_reqs;
 	/* statistics */
 	unsigned long		st_print;
 	int			st_rd_req;
 	int			st_wr_req;
 	int			st_oo_req;
 	int			st_f_req;
 	int			st_ds_req;
 	int			st_rd_sect;
 	int			st_wr_sect;
 	wait_queue_head_t	waiting_to_free;
 };
 #define vbd_sz(_v)	((_v)->bdev->bd_part ? \
 			 (_v)->bdev->bd_part->nr_sects : \
 			  get_capacity((_v)->bdev->bd_disk))
 #define xen_blkif_get(_b) (atomic_inc(&(_b)->refcnt))
 #define xen_blkif_put(_b)				\
 	do {						\
 		if (atomic_dec_and_test(&(_b)->refcnt))	\
 			wake_up(&(_b)->waiting_to_free);\
 	} while (0)
 struct phys_req {
 	unsigned short		dev;
 	blkif_sector_t		nr_sects;
 	struct block_device	*bdev;
 	blkif_sector_t		sector_number;
 };
 int xen_blkif_interface_init(void);
 int xen_blkif_xenbus_init(void);
 irqreturn_t xen_blkif_be_int(int irq, void *dev_id);
 int xen_blkif_schedule(void *arg);
 int xen_blkbk_flush_diskcache(struct xenbus_transaction xbt,
 			      struct backend_info *be, int state);
 int xen_blkbk_barrier(struct xenbus_transaction xbt,
 		      struct backend_info *be, int state);
 struct xenbus_device *xen_blkbk_xenbus(struct backend_info *be);
 static inline void blkif_get_x86_32_req(struct blkif_request *dst,
 					struct blkif_x86_32_request *src)
 {
 	int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
 	dst->operation = src->operation;
-	dst->nr_segments = src->nr_segments;
-	dst->handle = src->handle;
-	dst->id = src->id;
 	switch (src->operation) {
 	case BLKIF_OP_READ:
 	case BLKIF_OP_WRITE:
 	case BLKIF_OP_WRITE_BARRIER:
 	case BLKIF_OP_FLUSH_DISKCACHE:
+		dst->u.rw.nr_segments = src->u.rw.nr_segments;
+		dst->u.rw.handle = src->u.rw.handle;
+		dst->u.rw.id = src->u.rw.id;
 		dst->u.rw.sector_number = src->u.rw.sector_number;
 		barrier();
-		if (n > dst->nr_segments)
+		if (n > dst->u.rw.nr_segments)
-			n = dst->nr_segments;
+			n = dst->u.rw.nr_segments;
 		for (i = 0; i < n; i++)
 			dst->u.rw.seg[i] = src->u.rw.seg[i];
 		break;
 	case BLKIF_OP_DISCARD:
+		dst->u.discard.flag = src->u.discard.flag;
 		dst->u.discard.sector_number = src->u.discard.sector_number;
 		dst->u.discard.nr_sectors = src->u.discard.nr_sectors;
 		break;
 	default:
 		break;
 	}
 }
 static inline void blkif_get_x86_64_req(struct blkif_request *dst,
 					struct blkif_x86_64_request *src)
 {
 	int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
 	dst->operation = src->operation;
-	dst->nr_segments = src->nr_segments;
-	dst->handle = src->handle;
-	dst->id = src->id;
 	switch (src->operation) {
 	case BLKIF_OP_READ:
 	case BLKIF_OP_WRITE:
 	case BLKIF_OP_WRITE_BARRIER:
 	case BLKIF_OP_FLUSH_DISKCACHE:
+		dst->u.rw.nr_segments = src->u.rw.nr_segments;
+		dst->u.rw.handle = src->u.rw.handle;
+		dst->u.rw.id = src->u.rw.id;
 		dst->u.rw.sector_number = src->u.rw.sector_number;
 		barrier();
-		if (n > dst->nr_segments)
+		if (n > dst->u.rw.nr_segments)

drivers/block/xen-blkback/xenbus.c

Diff comments View file @ 16008d6

 /*  Xenbus code for blkif backend
     Copyright (C) 2005 Rusty Russell <rusty@rustcorp.com.au>
     Copyright (C) 2005 XenSource Ltd
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
 */
 #include <stdarg.h>
 #include <linux/module.h>
 #include <linux/kthread.h>
 #include <xen/events.h>
 #include <xen/grant_table.h>
 #include "common.h"
 struct backend_info {
 	struct xenbus_device	*dev;
 	struct xen_blkif	*blkif;
 	struct xenbus_watch	backend_watch;
 	unsigned		major;
 	unsigned		minor;
 	char			*mode;
 };
 static struct kmem_cache *xen_blkif_cachep;
 static void connect(struct backend_info *);
 static int connect_ring(struct backend_info *);
 static void backend_changed(struct xenbus_watch *, const char **,
 			    unsigned int);
 struct xenbus_device *xen_blkbk_xenbus(struct backend_info *be)
 {
 	return be->dev;
 }
 static int blkback_name(struct xen_blkif *blkif, char *buf)
 {
 	char *devpath, *devname;
 	struct xenbus_device *dev = blkif->be->dev;
 	devpath = xenbus_read(XBT_NIL, dev->nodename, "dev", NULL);
 	if (IS_ERR(devpath))
 		return PTR_ERR(devpath);
 	devname = strstr(devpath, "/dev/");
 	if (devname != NULL)
 		devname += strlen("/dev/");
 	else
 		devname  = devpath;
 	snprintf(buf, TASK_COMM_LEN, "blkback.%d.%s", blkif->domid, devname);
 	kfree(devpath);
 	return 0;
 }
 static void xen_update_blkif_status(struct xen_blkif *blkif)
 {
 	int err;
 	char name[TASK_COMM_LEN];
 	/* Not ready to connect? */
 	if (!blkif->irq || !blkif->vbd.bdev)
 		return;
 	/* Already connected? */
 	if (blkif->be->dev->state == XenbusStateConnected)
 		return;
 	/* Attempt to connect: exit if we fail to. */
 	connect(blkif->be);
 	if (blkif->be->dev->state != XenbusStateConnected)
 		return;
 	err = blkback_name(blkif, name);
 	if (err) {
 		xenbus_dev_error(blkif->be->dev, err, "get blkback dev name");
 		return;
 	}
 	err = filemap_write_and_wait(blkif->vbd.bdev->bd_inode->i_mapping);
 	if (err) {
 		xenbus_dev_error(blkif->be->dev, err, "block flush");
 		return;
 	}
 	invalidate_inode_pages2(blkif->vbd.bdev->bd_inode->i_mapping);
 	blkif->xenblkd = kthread_run(xen_blkif_schedule, blkif, name);
 	if (IS_ERR(blkif->xenblkd)) {
 		err = PTR_ERR(blkif->xenblkd);
 		blkif->xenblkd = NULL;
 		xenbus_dev_error(blkif->be->dev, err, "start xenblkd");
 	}
 }
 static struct xen_blkif *xen_blkif_alloc(domid_t domid)
 {
 	struct xen_blkif *blkif;
 	blkif = kmem_cache_alloc(xen_blkif_cachep, GFP_KERNEL);
 	if (!blkif)
 		return ERR_PTR(-ENOMEM);
 	memset(blkif, 0, sizeof(*blkif));
 	blkif->domid = domid;
 	spin_lock_init(&blkif->blk_ring_lock);
 	atomic_set(&blkif->refcnt, 1);
 	init_waitqueue_head(&blkif->wq);
 	init_completion(&blkif->drain_complete);
 	atomic_set(&blkif->drain, 0);
 	blkif->st_print = jiffies;
 	init_waitqueue_head(&blkif->waiting_to_free);
 	return blkif;
 }
 static int xen_blkif_map(struct xen_blkif *blkif, unsigned long shared_page,
 			 unsigned int evtchn)
 {
 	int err;
 	/* Already connected through? */
 	if (blkif->irq)
 		return 0;
 	err = xenbus_map_ring_valloc(blkif->be->dev, shared_page, &blkif->blk_ring);
 	if (err < 0)
 		return err;
 	switch (blkif->blk_protocol) {
 	case BLKIF_PROTOCOL_NATIVE:
 	{
 		struct blkif_sring *sring;
 		sring = (struct blkif_sring *)blkif->blk_ring;
 		BACK_RING_INIT(&blkif->blk_rings.native, sring, PAGE_SIZE);
 		break;
 	}
 	case BLKIF_PROTOCOL_X86_32:
 	{
 		struct blkif_x86_32_sring *sring_x86_32;
 		sring_x86_32 = (struct blkif_x86_32_sring *)blkif->blk_ring;
 		BACK_RING_INIT(&blkif->blk_rings.x86_32, sring_x86_32, PAGE_SIZE);
 		break;
 	}
 	case BLKIF_PROTOCOL_X86_64:
 	{
 		struct blkif_x86_64_sring *sring_x86_64;
 		sring_x86_64 = (struct blkif_x86_64_sring *)blkif->blk_ring;
 		BACK_RING_INIT(&blkif->blk_rings.x86_64, sring_x86_64, PAGE_SIZE);
 		break;
 	}
 	default:
 		BUG();
 	}
 	err = bind_interdomain_evtchn_to_irqhandler(blkif->domid, evtchn,
 						    xen_blkif_be_int, 0,
 						    "blkif-backend", blkif);
 	if (err < 0) {
 		xenbus_unmap_ring_vfree(blkif->be->dev, blkif->blk_ring);
 		blkif->blk_rings.common.sring = NULL;
 		return err;
 	}
 	blkif->irq = err;
 	return 0;
 }
 static void xen_blkif_disconnect(struct xen_blkif *blkif)
 {
 	if (blkif->xenblkd) {
 		kthread_stop(blkif->xenblkd);
 		blkif->xenblkd = NULL;
 	}
 	atomic_dec(&blkif->refcnt);
 	wait_event(blkif->waiting_to_free, atomic_read(&blkif->refcnt) == 0);
 	atomic_inc(&blkif->refcnt);
 	if (blkif->irq) {
 		unbind_from_irqhandler(blkif->irq, blkif);
 		blkif->irq = 0;
 	}
 	if (blkif->blk_rings.common.sring) {
 		xenbus_unmap_ring_vfree(blkif->be->dev, blkif->blk_ring);
 		blkif->blk_rings.common.sring = NULL;
 	}
 }
 void xen_blkif_free(struct xen_blkif *blkif)
 {
 	if (!atomic_dec_and_test(&blkif->refcnt))
 		BUG();
 	kmem_cache_free(xen_blkif_cachep, blkif);
 }
 int __init xen_blkif_interface_init(void)
 {
 	xen_blkif_cachep = kmem_cache_create("blkif_cache",
 					     sizeof(struct xen_blkif),
 					     0, 0, NULL);
 	if (!xen_blkif_cachep)
 		return -ENOMEM;
 	return 0;
 }
 /*
  *  sysfs interface for VBD I/O requests
  */
 #define VBD_SHOW(name, format, args...)					\
 	static ssize_t show_##name(struct device *_dev,			\
 				   struct device_attribute *attr,	\
 				   char *buf)				\
 	{								\
 		struct xenbus_device *dev = to_xenbus_device(_dev);	\
 		struct backend_info *be = dev_get_drvdata(&dev->dev);	\
 									\
 		return sprintf(buf, format, ##args);			\
 	}								\
 	static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 VBD_SHOW(oo_req,  "%d\n", be->blkif->st_oo_req);
 VBD_SHOW(rd_req,  "%d\n", be->blkif->st_rd_req);
 VBD_SHOW(wr_req,  "%d\n", be->blkif->st_wr_req);
 VBD_SHOW(f_req,  "%d\n", be->blkif->st_f_req);
 VBD_SHOW(ds_req,  "%d\n", be->blkif->st_ds_req);
 VBD_SHOW(rd_sect, "%d\n", be->blkif->st_rd_sect);
 VBD_SHOW(wr_sect, "%d\n", be->blkif->st_wr_sect);
 static struct attribute *xen_vbdstat_attrs[] = {
 	&dev_attr_oo_req.attr,
 	&dev_attr_rd_req.attr,
 	&dev_attr_wr_req.attr,
 	&dev_attr_f_req.attr,
 	&dev_attr_ds_req.attr,
 	&dev_attr_rd_sect.attr,
 	&dev_attr_wr_sect.attr,
 	NULL
 };
 static struct attribute_group xen_vbdstat_group = {
 	.name = "statistics",
 	.attrs = xen_vbdstat_attrs,
 };
 VBD_SHOW(physical_device, "%x:%x\n", be->major, be->minor);
 VBD_SHOW(mode, "%s\n", be->mode);
 int xenvbd_sysfs_addif(struct xenbus_device *dev)
 {
 	int error;
 	error = device_create_file(&dev->dev, &dev_attr_physical_device);
 	if (error)
 		goto fail1;
 	error = device_create_file(&dev->dev, &dev_attr_mode);
 	if (error)
 		goto fail2;
 	error = sysfs_create_group(&dev->dev.kobj, &xen_vbdstat_group);
 	if (error)
 		goto fail3;
 	return 0;
 fail3:	sysfs_remove_group(&dev->dev.kobj, &xen_vbdstat_group);
 fail2:	device_remove_file(&dev->dev, &dev_attr_mode);
 fail1:	device_remove_file(&dev->dev, &dev_attr_physical_device);
 	return error;
 }
 void xenvbd_sysfs_delif(struct xenbus_device *dev)
 {
 	sysfs_remove_group(&dev->dev.kobj, &xen_vbdstat_group);
 	device_remove_file(&dev->dev, &dev_attr_mode);
 	device_remove_file(&dev->dev, &dev_attr_physical_device);
 }
 static void xen_vbd_free(struct xen_vbd *vbd)
 {
 	if (vbd->bdev)
 		blkdev_put(vbd->bdev, vbd->readonly ? FMODE_READ : FMODE_WRITE);
 	vbd->bdev = NULL;
 }
 static int xen_vbd_create(struct xen_blkif *blkif, blkif_vdev_t handle,
 			  unsigned major, unsigned minor, int readonly,
 			  int cdrom)
 {
 	struct xen_vbd *vbd;
 	struct block_device *bdev;
 	struct request_queue *q;
 	vbd = &blkif->vbd;
 	vbd->handle   = handle;
 	vbd->readonly = readonly;
 	vbd->type     = 0;
 	vbd->pdevice  = MKDEV(major, minor);
 	bdev = blkdev_get_by_dev(vbd->pdevice, vbd->readonly ?
 				 FMODE_READ : FMODE_WRITE, NULL);
 	if (IS_ERR(bdev)) {
 		DPRINTK("xen_vbd_create: device %08x could not be opened.\n",
 			vbd->pdevice);
 		return -ENOENT;
 	}
 	vbd->bdev = bdev;
 	if (vbd->bdev->bd_disk == NULL) {
 		DPRINTK("xen_vbd_create: device %08x doesn't exist.\n",
 			vbd->pdevice);
 		xen_vbd_free(vbd);
 		return -ENOENT;
 	}
 	vbd->size = vbd_sz(vbd);
 	if (vbd->bdev->bd_disk->flags & GENHD_FL_CD || cdrom)
 		vbd->type |= VDISK_CDROM;
 	if (vbd->bdev->bd_disk->flags & GENHD_FL_REMOVABLE)
 		vbd->type |= VDISK_REMOVABLE;
 	q = bdev_get_queue(bdev);
 	if (q && q->flush_flags)
 		vbd->flush_support = true;
+	if (q && blk_queue_secdiscard(q))
+		vbd->discard_secure = true;
 	DPRINTK("Successful creation of handle=%04x (dom=%u)\n",
 		handle, blkif->domid);
 	return 0;
 }
 static int xen_blkbk_remove(struct xenbus_device *dev)
 {
 	struct backend_info *be = dev_get_drvdata(&dev->dev);
 	DPRINTK("");
 	if (be->major || be->minor)
 		xenvbd_sysfs_delif(dev);
 	if (be->backend_watch.node) {
 		unregister_xenbus_watch(&be->backend_watch);
 		kfree(be->backend_watch.node);
 		be->backend_watch.node = NULL;
 	}
 	if (be->blkif) {
 		xen_blkif_disconnect(be->blkif);
 		xen_vbd_free(&be->blkif->vbd);
 		xen_blkif_free(be->blkif);
 		be->blkif = NULL;
 	}
 	kfree(be);
 	dev_set_drvdata(&dev->dev, NULL);
 	return 0;
 }
 int xen_blkbk_flush_diskcache(struct xenbus_transaction xbt,
 			      struct backend_info *be, int state)
 {
 	struct xenbus_device *dev = be->dev;
 	int err;
 	err = xenbus_printf(xbt, dev->nodename, "feature-flush-cache",
 			    "%d", state);
 	if (err)
 		xenbus_dev_fatal(dev, err, "writing feature-flush-cache");
 	return err;
 }
 int xen_blkbk_discard(struct xenbus_transaction xbt, struct backend_info *be)
 {
 	struct xenbus_device *dev = be->dev;
 	struct xen_blkif *blkif = be->blkif;
 	char *type;
 	int err;
 	int state = 0;
 	type = xenbus_read(XBT_NIL, dev->nodename, "type", NULL);
 	if (!IS_ERR(type)) {
 		if (strncmp(type, "file", 4) == 0) {
 			state = 1;
 			blkif->blk_backend_type = BLKIF_BACKEND_FILE;
 		}
 		if (strncmp(type, "phy", 3) == 0) {
 			struct block_device *bdev = be->blkif->vbd.bdev;
 			struct request_queue *q = bdev_get_queue(bdev);
 			if (blk_queue_discard(q)) {
 				err = xenbus_printf(xbt, dev->nodename,
 					"discard-granularity", "%u",
 					q->limits.discard_granularity);
 				if (err) {
 					xenbus_dev_fatal(dev, err,
 						"writing discard-granularity");
 					goto kfree;
 				}
 				err = xenbus_printf(xbt, dev->nodename,
 					"discard-alignment", "%u",
 					q->limits.discard_alignment);
 				if (err) {
 					xenbus_dev_fatal(dev, err,
 						"writing discard-alignment");
 					goto kfree;
 				}
 				state = 1;
 				blkif->blk_backend_type = BLKIF_BACKEND_PHY;
+			}
+			/* Optional. */
+			err = xenbus_printf(xbt, dev->nodename,
+				"discard-secure", "%d",
+				blkif->vbd.discard_secure);
+			if (err) {
+				xenbus_dev_fatal(dev, err,
+					"writting discard-secure");
+				goto kfree;
 			}
 		}
 	} else {
 		err = PTR_ERR(type);
 		xenbus_dev_fatal(dev, err, "reading type");
 		goto out;
 	}
 	err = xenbus_printf(xbt, dev->nodename, "feature-discard",
 			    "%d", state);
 	if (err)
 		xenbus_dev_fatal(dev, err, "writing feature-discard");
 kfree:
 	kfree(type);
 out:
 	return err;
 }
 int xen_blkbk_barrier(struct xenbus_transaction xbt,
 		      struct backend_info *be, int state)
 {
 	struct xenbus_device *dev = be->dev;
 	int err;
 	err = xenbus_printf(xbt, dev->nodename, "feature-barrier",
 			    "%d", state);
 	if (err)
 		xenbus_dev_fatal(dev, err, "writing feature-barrier");
 	return err;
 }
 /*
  * Entry point to this code when a new device is created.  Allocate the basic
  * structures, and watch the store waiting for the hotplug scripts to tell us
  * the device's physical major and minor numbers.  Switch to InitWait.
  */
 static int xen_blkbk_probe(struct xenbus_device *dev,
 			   const struct xenbus_device_id *id)
 {
 	int err;
 	struct backend_info *be = kzalloc(sizeof(struct backend_info),
 					  GFP_KERNEL);
 	if (!be) {
 		xenbus_dev_fatal(dev, -ENOMEM,
 				 "allocating backend structure");
 		return -ENOMEM;
 	}
 	be->dev = dev;
 	dev_set_drvdata(&dev->dev, be);
 	be->blkif = xen_blkif_alloc(dev->otherend_id);
 	if (IS_ERR(be->blkif)) {
 		err = PTR_ERR(be->blkif);
 		be->blkif = NULL;
 		xenbus_dev_fatal(dev, err, "creating block interface");
 		goto fail;
 	}
 	/* setup back pointer */
 	be->blkif->be = be;
 	err = xenbus_watch_pathfmt(dev, &be->backend_watch, backend_changed,
 				   "%s/%s", dev->nodename, "physical-device");
 	if (err)
 		goto fail;
 	err = xenbus_switch_state(dev, XenbusStateInitWait);
 	if (err)
 		goto fail;
 	return 0;
 fail:
 	DPRINTK("failed");
 	xen_blkbk_remove(dev);
 	return err;
 }
 /*
  * Callback received when the hotplug scripts have placed the physical-device
  * node.  Read it and the mode node, and create a vbd.  If the frontend is
  * ready, connect.
  */
 static void backend_changed(struct xenbus_watch *watch,
 			    const char **vec, unsigned int len)
 {
 	int err;
 	unsigned major;
 	unsigned minor;
 	struct backend_info *be
 		= container_of(watch, struct backend_info, backend_watch);
 	struct xenbus_device *dev = be->dev;
 	int cdrom = 0;
 	char *device_type;
 	DPRINTK("");
 	err = xenbus_scanf(XBT_NIL, dev->nodename, "physical-device", "%x:%x",
 			   &major, &minor);
 	if (XENBUS_EXIST_ERR(err)) {
 		/*
 		 * Since this watch will fire once immediately after it is
 		 * registered, we expect this.  Ignore it, and wait for the
 		 * hotplug scripts.
 		 */
 		return;
 	}
 	if (err != 2) {
 		xenbus_dev_fatal(dev, err, "reading physical-device");
 		return;
 	}
 	if ((be->major || be->minor) &&
 	    ((be->major != major) || (be->minor != minor))) {
 		pr_warn(DRV_PFX "changing physical device (from %x:%x to %x:%x) not supported.\n",
 			be->major, be->minor, major, minor);
 		return;
 	}
 	be->mode = xenbus_read(XBT_NIL, dev->nodename, "mode", NULL);
 	if (IS_ERR(be->mode)) {
 		err = PTR_ERR(be->mode);
 		be->mode = NULL;
 		xenbus_dev_fatal(dev, err, "reading mode");
 		return;
 	}
 	device_type = xenbus_read(XBT_NIL, dev->otherend, "device-type", NULL);
 	if (!IS_ERR(device_type)) {
 		cdrom = strcmp(device_type, "cdrom") == 0;
 		kfree(device_type);
 	}
 	if (be->major == 0 && be->minor == 0) {
 		/* Front end dir is a number, which is used as the handle. */
 		char *p = strrchr(dev->otherend, '/') + 1;
 		long handle;
 		err = strict_strtoul(p, 0, &handle);
 		if (err)
 			return;
 		be->major = major;
 		be->minor = minor;
 		err = xen_vbd_create(be->blkif, handle, major, minor,
 				 (NULL == strchr(be->mode, 'w')), cdrom);
 		if (err) {
 			be->major = 0;
 			be->minor = 0;
 			xenbus_dev_fatal(dev, err, "creating vbd structure");
 			return;
 		}
 		err = xenvbd_sysfs_addif(dev);
 		if (err) {
 			xen_vbd_free(&be->blkif->vbd);
 			be->major = 0;
 			be->minor = 0;
 			xenbus_dev_fatal(dev, err, "creating sysfs entries");
 			return;
 		}
 		/* We're potentially connected now */
 		xen_update_blkif_status(be->blkif);
 	}
 }
 /*
  * Callback received when the frontend's state changes.
  */
 static void frontend_changed(struct xenbus_device *dev,
 			     enum xenbus_state frontend_state)
 {
 	struct backend_info *be = dev_get_drvdata(&dev->dev);
 	int err;
 	DPRINTK("%s", xenbus_strstate(frontend_state));
 	switch (frontend_state) {
 	case XenbusStateInitialising:
 		if (dev->state == XenbusStateClosed) {
 			pr_info(DRV_PFX "%s: prepare for reconnect\n",
 				dev->nodename);
 			xenbus_switch_state(dev, XenbusStateInitWait);
 		}
 		break;
 	case XenbusStateInitialised:
 	case XenbusStateConnected:
 		/*
 		 * Ensure we connect even when two watches fire in
 		 * close succession and we miss the intermediate value
 		 * of frontend_state.
 		 */
 		if (dev->state == XenbusStateConnected)
 			break;
 		/*
 		 * Enforce precondition before potential leak point.
 		 * xen_blkif_disconnect() is idempotent.
 		 */
 		xen_blkif_disconnect(be->blkif);
 		err = connect_ring(be);
 		if (err)
 			break;
 		xen_update_blkif_status(be->blkif);
 		break;
 	case XenbusStateClosing:
 		xenbus_switch_state(dev, XenbusStateClosing);
 		break;
 	case XenbusStateClosed:
 		xen_blkif_disconnect(be->blkif);
 		xenbus_switch_state(dev, XenbusStateClosed);
 		if (xenbus_dev_is_online(dev))
 			break;
 		/* fall through if not online */
 	case XenbusStateUnknown:
 		/* implies xen_blkif_disconnect() via xen_blkbk_remove() */
 		device_unregister(&dev->dev);
 		break;
 	default:
 		xenbus_dev_fatal(dev, -EINVAL, "saw state %d at frontend",
 				 frontend_state);
 		break;
 	}
 }
 /* ** Connection ** */
 /*
  * Write the physical details regarding the block device to the store, and
  * switch to Connected state.
  */
 static void connect(struct backend_info *be)
 {
 	struct xenbus_transaction xbt;
 	int err;
 	struct xenbus_device *dev = be->dev;
 	DPRINTK("%s", dev->otherend);
 	/* Supply the information about the device the frontend needs */
 again:
 	err = xenbus_transaction_start(&xbt);
 	if (err) {
 		xenbus_dev_fatal(dev, err, "starting transaction");
 		return;
 	}
 	err = xen_blkbk_flush_diskcache(xbt, be, be->blkif->vbd.flush_support);
 	if (err)
 		goto abort;
 	err = xen_blkbk_discard(xbt, be);
 	/* If we can't advertise it is OK. */
 	err = xen_blkbk_barrier(xbt, be, be->blkif->vbd.flush_support);
 	err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu",
 			    (unsigned long long)vbd_sz(&be->blkif->vbd));
 	if (err) {
 		xenbus_dev_fatal(dev, err, "writing %s/sectors",
 				 dev->nodename);
 		goto abort;
 	}
 	/* FIXME: use a typename instead */
 	err = xenbus_printf(xbt, dev->nodename, "info", "%u",
 			    be->blkif->vbd.type |
 			    (be->blkif->vbd.readonly ? VDISK_READONLY : 0));
 	if (err) {
 		xenbus_dev_fatal(dev, err, "writing %s/info",
 				 dev->nodename);
 		goto abort;
 	}
 	err = xenbus_printf(xbt, dev->nodename, "sector-size", "%lu",
 			    (unsigned long)
 			    bdev_logical_block_size(be->blkif->vbd.bdev));
 	if (err) {
 		xenbus_dev_fatal(dev, err, "writing %s/sector-size",
 				 dev->nodename);
 		goto abort;
 	}
 	err = xenbus_transaction_end(xbt, 0);
 	if (err == -EAGAIN)
 		goto again;
 	if (err)
 		xenbus_dev_fatal(dev, err, "ending transaction");
 	err = xenbus_switch_state(dev, XenbusStateConnected);
 	if (err)
 		xenbus_dev_fatal(dev, err, "%s: switching to Connected state",
 				 dev->nodename);
 	return;
  abort:
 	xenbus_transaction_end(xbt, 1);
 }
 static int connect_ring(struct backend_info *be)
 {
 	struct xenbus_device *dev = be->dev;
 	unsigned long ring_ref;
 	unsigned int evtchn;
 	char protocol[64] = "";
 	int err;
 	DPRINTK("%s", dev->otherend);
 	err = xenbus_gather(XBT_NIL, dev->otherend, "ring-ref", "%lu",
 			    &ring_ref, "event-channel", "%u", &evtchn, NULL);
 	if (err) {
 		xenbus_dev_fatal(dev, err,
 				 "reading %s/ring-ref and event-channel",
 				 dev->otherend);
 		return err;
 	}
 	be->blkif->blk_protocol = BLKIF_PROTOCOL_NATIVE;
 	err = xenbus_gather(XBT_NIL, dev->otherend, "protocol",
 			    "%63s", protocol, NULL);
 	if (err)
 		strcpy(protocol, "unspecified, assuming native");
 	else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE))
 		be->blkif->blk_protocol = BLKIF_PROTOCOL_NATIVE;
 	else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_X86_32))
 		be->blkif->blk_protocol = BLKIF_PROTOCOL_X86_32;
 	else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_X86_64))
 		be->blkif->blk_protocol = BLKIF_PROTOCOL_X86_64;
 	else {
 		xenbus_dev_fatal(dev, err, "unknown fe protocol %s", protocol);
 		return -1;
 	}
 	pr_info(DRV_PFX "ring-ref %ld, event-channel %d, protocol %d (%s)\n",
 		ring_ref, evtchn, be->blkif->blk_protocol, protocol);
 	/* Map the shared frame, irq etc. */
 	err = xen_blkif_map(be->blkif, ring_ref, evtchn);
 	if (err) {
 		xenbus_dev_fatal(dev, err, "mapping ring-ref %lu port %u",
 				 ring_ref, evtchn);
 		return err;
 	}
 	return 0;
 }
 /* ** Driver Registration ** */
 static const struct xenbus_device_id xen_blkbk_ids[] = {
 	{ "vbd" },
 	{ "" }
 };
 static DEFINE_XENBUS_DRIVER(xen_blkbk, ,
 	.probe = xen_blkbk_probe,
 	.remove = xen_blkbk_remove,
 	.otherend_changed = frontend_changed
 );
 int xen_blkif_xenbus_init(void)
 {
 	return xenbus_register_backend(&xen_blkbk_driver);
 }

drivers/block/xen-blkfront.c

Diff comments View file @ 16008d6

 /*
  * blkfront.c
  *
  * XenLinux virtual block device driver.
  *
  * Copyright (c) 2003-2004, Keir Fraser & Steve Hand
  * Modifications by Mark A. Williamson are (c) Intel Research Cambridge
  * Copyright (c) 2004, Christian Limpach
  * Copyright (c) 2004, Andrew Warfield
  * Copyright (c) 2005, Christopher Clark
  * Copyright (c) 2005, XenSource Ltd
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License version 2
  * as published by the Free Software Foundation; or, when distributed
  * separately from the Linux kernel or incorporated into other
  * software packages, subject to the following license:
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this source file (the "Software"), to deal in the Software without
  * restriction, including without limitation the rights to use, copy, modify,
  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  * and to permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in
  * all copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  * IN THE SOFTWARE.
  */
 #include <linux/interrupt.h>
 #include <linux/blkdev.h>
 #include <linux/hdreg.h>
 #include <linux/cdrom.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/mutex.h>
 #include <linux/scatterlist.h>
 #include <xen/xen.h>
 #include <xen/xenbus.h>
 #include <xen/grant_table.h>
 #include <xen/events.h>
 #include <xen/page.h>
 #include <xen/platform_pci.h>
 #include <xen/interface/grant_table.h>
 #include <xen/interface/io/blkif.h>
 #include <xen/interface/io/protocols.h>
 #include <asm/xen/hypervisor.h>
 enum blkif_state {
 	BLKIF_STATE_DISCONNECTED,
 	BLKIF_STATE_CONNECTED,
 	BLKIF_STATE_SUSPENDED,
 };
 struct blk_shadow {
 	struct blkif_request req;
 	struct request *request;
 	unsigned long frame[BLKIF_MAX_SEGMENTS_PER_REQUEST];
 };
 static DEFINE_MUTEX(blkfront_mutex);
 static const struct block_device_operations xlvbd_block_fops;
 #define BLK_RING_SIZE __CONST_RING_SIZE(blkif, PAGE_SIZE)
 /*
  * We have one of these per vbd, whether ide, scsi or 'other'.  They
  * hang in private_data off the gendisk structure. We may end up
  * putting all kinds of interesting stuff here :-)
  */
 struct blkfront_info
 {
 	struct mutex mutex;
 	struct xenbus_device *xbdev;
 	struct gendisk *gd;
 	int vdevice;
 	blkif_vdev_t handle;
 	enum blkif_state connected;
 	int ring_ref;
 	struct blkif_front_ring ring;
 	struct scatterlist sg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
 	unsigned int evtchn, irq;
 	struct request_queue *rq;
 	struct work_struct work;
 	struct gnttab_free_callback callback;
 	struct blk_shadow shadow[BLK_RING_SIZE];
 	unsigned long shadow_free;
 	unsigned int feature_flush;
 	unsigned int flush_op;
-	unsigned int feature_discard;
+	unsigned int feature_discard:1;
+	unsigned int feature_secdiscard:1;
 	unsigned int discard_granularity;
 	unsigned int discard_alignment;
 	int is_ready;
 };
 static DEFINE_SPINLOCK(blkif_io_lock);
 static unsigned int nr_minors;
 static unsigned long *minors;
 static DEFINE_SPINLOCK(minor_lock);
 #define MAXIMUM_OUTSTANDING_BLOCK_REQS \
 	(BLKIF_MAX_SEGMENTS_PER_REQUEST * BLK_RING_SIZE)
 #define GRANT_INVALID_REF	0
 #define PARTS_PER_DISK		16
 #define PARTS_PER_EXT_DISK      256
 #define BLKIF_MAJOR(dev) ((dev)>>8)
 #define BLKIF_MINOR(dev) ((dev) & 0xff)
 #define EXT_SHIFT 28
 #define EXTENDED (1<<EXT_SHIFT)
 #define VDEV_IS_EXTENDED(dev) ((dev)&(EXTENDED))
 #define BLKIF_MINOR_EXT(dev) ((dev)&(~EXTENDED))
 #define EMULATED_HD_DISK_MINOR_OFFSET (0)
 #define EMULATED_HD_DISK_NAME_OFFSET (EMULATED_HD_DISK_MINOR_OFFSET / 256)
 #define EMULATED_SD_DISK_MINOR_OFFSET (0)
 #define EMULATED_SD_DISK_NAME_OFFSET (EMULATED_SD_DISK_MINOR_OFFSET / 256)
 #define DEV_NAME	"xvd"	/* name in /dev */
 static int get_id_from_freelist(struct blkfront_info *info)
 {
 	unsigned long free = info->shadow_free;
 	BUG_ON(free >= BLK_RING_SIZE);
-	info->shadow_free = info->shadow[free].req.id;
+	info->shadow_free = info->shadow[free].req.u.rw.id;
-	info->shadow[free].req.id = 0x0fffffee; /* debug */
+	info->shadow[free].req.u.rw.id = 0x0fffffee; /* debug */
 	return free;
 }
 static void add_id_to_freelist(struct blkfront_info *info,
 			       unsigned long id)
 {
-	info->shadow[id].req.id  = info->shadow_free;
+	info->shadow[id].req.u.rw.id  = info->shadow_free;
 	info->shadow[id].request = NULL;
 	info->shadow_free = id;
 }
 static int xlbd_reserve_minors(unsigned int minor, unsigned int nr)
 {
 	unsigned int end = minor + nr;
 	int rc;
 	if (end > nr_minors) {
 		unsigned long *bitmap, *old;
-		bitmap = kzalloc(BITS_TO_LONGS(end) * sizeof(*bitmap),
+		bitmap = kcalloc(BITS_TO_LONGS(end), sizeof(*bitmap),
 				 GFP_KERNEL);
 		if (bitmap == NULL)
 			return -ENOMEM;
 		spin_lock(&minor_lock);
 		if (end > nr_minors) {
 			old = minors;
 			memcpy(bitmap, minors,
 			       BITS_TO_LONGS(nr_minors) * sizeof(*bitmap));
 			minors = bitmap;
 			nr_minors = BITS_TO_LONGS(end) * BITS_PER_LONG;
 		} else
 			old = bitmap;
 		spin_unlock(&minor_lock);
 		kfree(old);
 	}
 	spin_lock(&minor_lock);
 	if (find_next_bit(minors, end, minor) >= end) {
 		for (; minor < end; ++minor)
 			__set_bit(minor, minors);
 		rc = 0;
 	} else
 		rc = -EBUSY;
 	spin_unlock(&minor_lock);
 	return rc;
 }
 static void xlbd_release_minors(unsigned int minor, unsigned int nr)
 {
 	unsigned int end = minor + nr;
 	BUG_ON(end > nr_minors);
 	spin_lock(&minor_lock);
 	for (; minor < end; ++minor)
 		__clear_bit(minor, minors);
 	spin_unlock(&minor_lock);
 }
 static void blkif_restart_queue_callback(void *arg)
 {
 	struct blkfront_info *info = (struct blkfront_info *)arg;
 	schedule_work(&info->work);
 }
 static int blkif_getgeo(struct block_device *bd, struct hd_geometry *hg)
 {
 	/* We don't have real geometry info, but let's at least return
 	   values consistent with the size of the device */
 	sector_t nsect = get_capacity(bd->bd_disk);
 	sector_t cylinders = nsect;
 	hg->heads = 0xff;
 	hg->sectors = 0x3f;
 	sector_div(cylinders, hg->heads * hg->sectors);
 	hg->cylinders = cylinders;
 	if ((sector_t)(hg->cylinders + 1) * hg->heads * hg->sectors < nsect)
 		hg->cylinders = 0xffff;
 	return 0;
 }
 static int blkif_ioctl(struct block_device *bdev, fmode_t mode,
 		       unsigned command, unsigned long argument)
 {
 	struct blkfront_info *info = bdev->bd_disk->private_data;
 	int i;
 	dev_dbg(&info->xbdev->dev, "command: 0x%x, argument: 0x%lx\n",
 		command, (long)argument);
 	switch (command) {
 	case CDROMMULTISESSION:
 		dev_dbg(&info->xbdev->dev, "FIXME: support multisession CDs later\n");
 		for (i = 0; i < sizeof(struct cdrom_multisession); i++)
 			if (put_user(0, (char __user *)(argument + i)))
 				return -EFAULT;
 		return 0;
 	case CDROM_GET_CAPABILITY: {
 		struct gendisk *gd = info->gd;
 		if (gd->flags & GENHD_FL_CD)
 			return 0;
 		return -EINVAL;
 	}
 	default:
 		/*printk(KERN_ALERT "ioctl %08x not supported by Xen blkdev\n",
 		  command);*/
 		return -EINVAL; /* same return as native Linux */
 	}
 	return 0;
 }
 /*
  * Generate a Xen blkfront IO request from a blk layer request.  Reads
  * and writes are handled as expected.
  *
  * @req: a request struct
  */
 static int blkif_queue_request(struct request *req)
 {
 	struct blkfront_info *info = req->rq_disk->private_data;
 	unsigned long buffer_mfn;
 	struct blkif_request *ring_req;
 	unsigned long id;
 	unsigned int fsect, lsect;
 	int i, ref;
 	grant_ref_t gref_head;
 	struct scatterlist *sg;
 	if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
 		return 1;
 	if (gnttab_alloc_grant_references(
 		BLKIF_MAX_SEGMENTS_PER_REQUEST, &gref_head) < 0) {
 		gnttab_request_free_callback(
 			&info->callback,
 			blkif_restart_queue_callback,
 			info,
 			BLKIF_MAX_SEGMENTS_PER_REQUEST);
 		return 1;
 	}
 	/* Fill out a communications ring structure. */
 	ring_req = RING_GET_REQUEST(&info->ring, info->ring.req_prod_pvt);
 	id = get_id_from_freelist(info);
 	info->shadow[id].request = req;
-	ring_req->id = id;
+	ring_req->u.rw.id = id;
 	ring_req->u.rw.sector_number = (blkif_sector_t)blk_rq_pos(req);
-	ring_req->handle = info->handle;
+	ring_req->u.rw.handle = info->handle;
 	ring_req->operation = rq_data_dir(req) ?
 		BLKIF_OP_WRITE : BLKIF_OP_READ;
 	if (req->cmd_flags & (REQ_FLUSH | REQ_FUA)) {
 		/*
 		 * Ideally we can do an unordered flush-to-disk. In case the
 		 * backend onlysupports barriers, use that. A barrier request
 		 * a superset of FUA, so we can implement it the same
 		 * way.  (It's also a FLUSH+FUA, since it is
 		 * guaranteed ordered WRT previous writes.)
 		 */
 		ring_req->operation = info->flush_op;
 	}
-	if (unlikely(req->cmd_flags & REQ_DISCARD)) {
+	if (unlikely(req->cmd_flags & (REQ_DISCARD | REQ_SECURE))) {
 		/* id, sector_number and handle are set above. */
 		ring_req->operation = BLKIF_OP_DISCARD;
-		ring_req->nr_segments = 0;
 		ring_req->u.discard.nr_sectors = blk_rq_sectors(req);
+		if ((req->cmd_flags & REQ_SECURE) && info->feature_secdiscard)
+			ring_req->u.discard.flag = BLKIF_DISCARD_SECURE;
+		else
+			ring_req->u.discard.flag = 0;
 	} else {
-		ring_req->nr_segments = blk_rq_map_sg(req->q, req, info->sg);
+		ring_req->u.rw.nr_segments = blk_rq_map_sg(req->q, req,
-		BUG_ON(ring_req->nr_segments > BLKIF_MAX_SEGMENTS_PER_REQUEST);
+							   info->sg);
+		BUG_ON(ring_req->u.rw.nr_segments >
+		       BLKIF_MAX_SEGMENTS_PER_REQUEST);
-		for_each_sg(info->sg, sg, ring_req->nr_segments, i) {
+		for_each_sg(info->sg, sg, ring_req->u.rw.nr_segments, i) {
 			buffer_mfn = pfn_to_mfn(page_to_pfn(sg_page(sg)));
 			fsect = sg->offset >> 9;
 			lsect = fsect + (sg->length >> 9) - 1;
 			/* install a grant reference. */
 			ref = gnttab_claim_grant_reference(&gref_head);
 			BUG_ON(ref == -ENOSPC);
 			gnttab_grant_foreign_access_ref(
 					ref,
 					info->xbdev->otherend_id,
 					buffer_mfn,
 					rq_data_dir(req));
 			info->shadow[id].frame[i] = mfn_to_pfn(buffer_mfn);
 			ring_req->u.rw.seg[i] =
 					(struct blkif_request_segment) {
 						.gref       = ref,
 						.first_sect = fsect,
 						.last_sect  = lsect };
 		}
 	}
 	info->ring.req_prod_pvt++;
 	/* Keep a private copy so we can reissue requests when recovering. */
 	info->shadow[id].req = *ring_req;
 	gnttab_free_grant_references(gref_head);
 	return 0;
 }
 static inline void flush_requests(struct blkfront_info *info)
 {
 	int notify;
 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&info->ring, notify);
 	if (notify)
 		notify_remote_via_irq(info->irq);
 }
 /*
  * do_blkif_request
  *  read a block; request is in a request queue
  */
 static void do_blkif_request(struct request_queue *rq)
 {
 	struct blkfront_info *info = NULL;
 	struct request *req;
 	int queued;
 	pr_debug("Entered do_blkif_request\n");
 	queued = 0;
 	while ((req = blk_peek_request(rq)) != NULL) {
 		info = req->rq_disk->private_data;
 		if (RING_FULL(&info->ring))
 			goto wait;
 		blk_start_request(req);
 		if ((req->cmd_type != REQ_TYPE_FS) ||
 		    ((req->cmd_flags & (REQ_FLUSH | REQ_FUA)) &&
 		    !info->flush_op)) {
 			__blk_end_request_all(req, -EIO);
 			continue;
 		}
 		pr_debug("do_blk_req %p: cmd %p, sec %lx, "
 			 "(%u/%u) buffer:%p [%s]\n",
 			 req, req->cmd, (unsigned long)blk_rq_pos(req),
 			 blk_rq_cur_sectors(req), blk_rq_sectors(req),
 			 req->buffer, rq_data_dir(req) ? "write" : "read");
 		if (blkif_queue_request(req)) {
 			blk_requeue_request(rq, req);
 wait:
 			/* Avoid pointless unplugs. */
 			blk_stop_queue(rq);
 			break;
 		}
 		queued++;
 	}
 	if (queued != 0)
 		flush_requests(info);
 }
 static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size)
 {
 	struct request_queue *rq;
 	struct blkfront_info *info = gd->private_data;
 	rq = blk_init_queue(do_blkif_request, &blkif_io_lock);
 	if (rq == NULL)
 		return -1;
 	queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq);
 	if (info->feature_discard) {
 		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, rq);
 		blk_queue_max_discard_sectors(rq, get_capacity(gd));
 		rq->limits.discard_granularity = info->discard_granularity;
 		rq->limits.discard_alignment = info->discard_alignment;
+		if (info->feature_secdiscard)
+			queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, rq);
 	}
 	/* Hard sector size and max sectors impersonate the equiv. hardware. */
 	blk_queue_logical_block_size(rq, sector_size);
 	blk_queue_max_hw_sectors(rq, 512);
 	/* Each segment in a request is up to an aligned page in size. */
 	blk_queue_segment_boundary(rq, PAGE_SIZE - 1);
 	blk_queue_max_segment_size(rq, PAGE_SIZE);
 	/* Ensure a merged request will fit in a single I/O ring slot. */
 	blk_queue_max_segments(rq, BLKIF_MAX_SEGMENTS_PER_REQUEST);
 	/* Make sure buffer addresses are sector-aligned. */
 	blk_queue_dma_alignment(rq, 511);
 	/* Make sure we don't use bounce buffers. */
 	blk_queue_bounce_limit(rq, BLK_BOUNCE_ANY);
 	gd->queue = rq;
 	return 0;
 }
 static void xlvbd_flush(struct blkfront_info *info)
 {
 	blk_queue_flush(info->rq, info->feature_flush);
 	printk(KERN_INFO "blkfront: %s: %s: %s\n",
 	       info->gd->disk_name,
 	       info->flush_op == BLKIF_OP_WRITE_BARRIER ?
 		"barrier" : (info->flush_op == BLKIF_OP_FLUSH_DISKCACHE ?
 		"flush diskcache" : "barrier or flush"),
 	       info->feature_flush ? "enabled" : "disabled");
 }
 static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset)
 {
 	int major;
 	major = BLKIF_MAJOR(vdevice);
 	*minor = BLKIF_MINOR(vdevice);
 	switch (major) {
 		case XEN_IDE0_MAJOR:
 			*offset = (*minor / 64) + EMULATED_HD_DISK_NAME_OFFSET;
 			*minor = ((*minor / 64) * PARTS_PER_DISK) +
 				EMULATED_HD_DISK_MINOR_OFFSET;
 			break;
 		case XEN_IDE1_MAJOR:
 			*offset = (*minor / 64) + 2 + EMULATED_HD_DISK_NAME_OFFSET;
 			*minor = (((*minor / 64) + 2) * PARTS_PER_DISK) +
 				EMULATED_HD_DISK_MINOR_OFFSET;
 			break;
 		case XEN_SCSI_DISK0_MAJOR:
 			*offset = (*minor / PARTS_PER_DISK) + EMULATED_SD_DISK_NAME_OFFSET;
 			*minor = *minor + EMULATED_SD_DISK_MINOR_OFFSET;
 			break;
 		case XEN_SCSI_DISK1_MAJOR:
 		case XEN_SCSI_DISK2_MAJOR:
 		case XEN_SCSI_DISK3_MAJOR:
 		case XEN_SCSI_DISK4_MAJOR:
 		case XEN_SCSI_DISK5_MAJOR:
 		case XEN_SCSI_DISK6_MAJOR:
 		case XEN_SCSI_DISK7_MAJOR:
 			*offset = (*minor / PARTS_PER_DISK) +
 				((major - XEN_SCSI_DISK1_MAJOR + 1) * 16) +
 				EMULATED_SD_DISK_NAME_OFFSET;
 			*minor = *minor +
 				((major - XEN_SCSI_DISK1_MAJOR + 1) * 16 * PARTS_PER_DISK) +
 				EMULATED_SD_DISK_MINOR_OFFSET;
 			break;
 		case XEN_SCSI_DISK8_MAJOR:
 		case XEN_SCSI_DISK9_MAJOR:
 		case XEN_SCSI_DISK10_MAJOR:
 		case XEN_SCSI_DISK11_MAJOR:
 		case XEN_SCSI_DISK12_MAJOR:
 		case XEN_SCSI_DISK13_MAJOR:
 		case XEN_SCSI_DISK14_MAJOR:
 		case XEN_SCSI_DISK15_MAJOR:
 			*offset = (*minor / PARTS_PER_DISK) +
 				((major - XEN_SCSI_DISK8_MAJOR + 8) * 16) +
 				EMULATED_SD_DISK_NAME_OFFSET;
 			*minor = *minor +
 				((major - XEN_SCSI_DISK8_MAJOR + 8) * 16 * PARTS_PER_DISK) +
 				EMULATED_SD_DISK_MINOR_OFFSET;
 			break;
 		case XENVBD_MAJOR:
 			*offset = *minor / PARTS_PER_DISK;
 			break;
 		default:
 			printk(KERN_WARNING "blkfront: your disk configuration is "
 					"incorrect, please use an xvd device instead\n");
 			return -ENODEV;
 	}
 	return 0;
 }
 static int xlvbd_alloc_gendisk(blkif_sector_t capacity,
 			       struct blkfront_info *info,
 			       u16 vdisk_info, u16 sector_size)
 {
 	struct gendisk *gd;
 	int nr_minors = 1;
 	int err;
 	unsigned int offset;
 	int minor;
 	int nr_parts;
 	BUG_ON(info->gd != NULL);
 	BUG_ON(info->rq != NULL);
 	if ((info->vdevice>>EXT_SHIFT) > 1) {
 		/* this is above the extended range; something is wrong */
 		printk(KERN_WARNING "blkfront: vdevice 0x%x is above the extended range; ignoring\n", info->vdevice);
 		return -ENODEV;
 	}
 	if (!VDEV_IS_EXTENDED(info->vdevice)) {
 		err = xen_translate_vdev(info->vdevice, &minor, &offset);
 		if (err)
 			return err;
  		nr_parts = PARTS_PER_DISK;
 	} else {
 		minor = BLKIF_MINOR_EXT(info->vdevice);
 		nr_parts = PARTS_PER_EXT_DISK;
 		offset = minor / nr_parts;
 		if (xen_hvm_domain() && offset < EMULATED_HD_DISK_NAME_OFFSET + 4)
 			printk(KERN_WARNING "blkfront: vdevice 0x%x might conflict with "
 					"emulated IDE disks,\n\t choose an xvd device name"
 					"from xvde on\n", info->vdevice);
 	}
 	err = -ENODEV;
 	if ((minor % nr_parts) == 0)
 		nr_minors = nr_parts;
 	err = xlbd_reserve_minors(minor, nr_minors);
 	if (err)
 		goto out;
 	err = -ENODEV;
 	gd = alloc_disk(nr_minors);
 	if (gd == NULL)
 		goto release;
 	if (nr_minors > 1) {
 		if (offset < 26)
 			sprintf(gd->disk_name, "%s%c", DEV_NAME, 'a' + offset);
 		else
 			sprintf(gd->disk_name, "%s%c%c", DEV_NAME,
 				'a' + ((offset / 26)-1), 'a' + (offset % 26));
 	} else {
 		if (offset < 26)
 			sprintf(gd->disk_name, "%s%c%d", DEV_NAME,
 				'a' + offset,
 				minor & (nr_parts - 1));
 		else
 			sprintf(gd->disk_name, "%s%c%c%d", DEV_NAME,
 				'a' + ((offset / 26) - 1),
 				'a' + (offset % 26),
 				minor & (nr_parts - 1));
 	}
 	gd->major = XENVBD_MAJOR;
 	gd->first_minor = minor;
 	gd->fops = &xlvbd_block_fops;
 	gd->private_data = info;
 	gd->driverfs_dev = &(info->xbdev->dev);
 	set_capacity(gd, capacity);
 	if (xlvbd_init_blk_queue(gd, sector_size)) {
 		del_gendisk(gd);
 		goto release;
 	}
 	info->rq = gd->queue;
 	info->gd = gd;
 	xlvbd_flush(info);
 	if (vdisk_info & VDISK_READONLY)
 		set_disk_ro(gd, 1);
 	if (vdisk_info & VDISK_REMOVABLE)
 		gd->flags |= GENHD_FL_REMOVABLE;
 	if (vdisk_info & VDISK_CDROM)
 		gd->flags |= GENHD_FL_CD;
 	return 0;
  release:
 	xlbd_release_minors(minor, nr_minors);
  out:
 	return err;
 }
 static void xlvbd_release_gendisk(struct blkfront_info *info)
 {
 	unsigned int minor, nr_minors;
 	unsigned long flags;
 	if (info->rq == NULL)
 		return;
 	spin_lock_irqsave(&blkif_io_lock, flags);
 	/* No more blkif_request(). */
 	blk_stop_queue(info->rq);
 	/* No more gnttab callback work. */
 	gnttab_cancel_free_callback(&info->callback);
 	spin_unlock_irqrestore(&blkif_io_lock, flags);
 	/* Flush gnttab callback work. Must be done with no locks held. */
 	flush_work_sync(&info->work);
 	del_gendisk(info->gd);
 	minor = info->gd->first_minor;
 	nr_minors = info->gd->minors;
 	xlbd_release_minors(minor, nr_minors);
 	blk_cleanup_queue(info->rq);
 	info->rq = NULL;
 	put_disk(info->gd);
 	info->gd = NULL;
 }
 static void kick_pending_request_queues(struct blkfront_info *info)
 {
 	if (!RING_FULL(&info->ring)) {
 		/* Re-enable calldowns. */
 		blk_start_queue(info->rq);
 		/* Kick things off immediately. */
 		do_blkif_request(info->rq);
 	}
 }
 static void blkif_restart_queue(struct work_struct *work)
 {
 	struct blkfront_info *info = container_of(work, struct blkfront_info, work);
 	spin_lock_irq(&blkif_io_lock);
 	if (info->connected == BLKIF_STATE_CONNECTED)
 		kick_pending_request_queues(info);
 	spin_unlock_irq(&blkif_io_lock);
 }
 static void blkif_free(struct blkfront_info *info, int suspend)
 {
 	/* Prevent new requests being issued until we fix things up. */
 	spin_lock_irq(&blkif_io_lock);
 	info->connected = suspend ?
 		BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED;
 	/* No more blkif_request(). */
 	if (info->rq)
 		blk_stop_queue(info->rq);
 	/* No more gnttab callback work. */
 	gnttab_cancel_free_callback(&info->callback);
 	spin_unlock_irq(&blkif_io_lock);
 	/* Flush gnttab callback work. Must be done with no locks held. */
 	flush_work_sync(&info->work);
 	/* Free resources associated with old device channel. */
 	if (info->ring_ref != GRANT_INVALID_REF) {
 		gnttab_end_foreign_access(info->ring_ref, 0,
 					  (unsigned long)info->ring.sring);
 		info->ring_ref = GRANT_INVALID_REF;
 		info->ring.sring = NULL;
 	}
 	if (info->irq)
 		unbind_from_irqhandler(info->irq, info);
 	info->evtchn = info->irq = 0;
 }
 static void blkif_completion(struct blk_shadow *s)
 {
 	int i;
-	for (i = 0; i < s->req.nr_segments; i++)
+	/* Do not let BLKIF_OP_DISCARD as nr_segment is in the same place
+	 * flag. */
+	for (i = 0; i < s->req.u.rw.nr_segments; i++)
 		gnttab_end_foreign_access(s->req.u.rw.seg[i].gref, 0, 0UL);
 }
 static irqreturn_t blkif_interrupt(int irq, void *dev_id)
 {
 	struct request *req;
 	struct blkif_response *bret;
 	RING_IDX i, rp;
 	unsigned long flags;
 	struct blkfront_info *info = (struct blkfront_info *)dev_id;
 	int error;
 	spin_lock_irqsave(&blkif_io_lock, flags);
 	if (unlikely(info->connected != BLKIF_STATE_CONNECTED)) {
 		spin_unlock_irqrestore(&blkif_io_lock, flags);
 		return IRQ_HANDLED;
 	}
  again:
 	rp = info->ring.sring->rsp_prod;
 	rmb(); /* Ensure we see queued responses up to 'rp'. */
 	for (i = info->ring.rsp_cons; i != rp; i++) {
 		unsigned long id;
 		bret = RING_GET_RESPONSE(&info->ring, i);
 		id   = bret->id;
 		req  = info->shadow[id].request;
-		blkif_completion(&info->shadow[id]);
+		if (bret->operation != BLKIF_OP_DISCARD)
+			blkif_completion(&info->shadow[id]);
 		add_id_to_freelist(info, id);
 		error = (bret->status == BLKIF_RSP_OKAY) ? 0 : -EIO;
 		switch (bret->operation) {
 		case BLKIF_OP_DISCARD:
 			if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
 				struct request_queue *rq = info->rq;
 				printk(KERN_WARNING "blkfront: %s: discard op failed\n",
 					   info->gd->disk_name);
 				error = -EOPNOTSUPP;
 				info->feature_discard = 0;
+				info->feature_secdiscard = 0;
 				queue_flag_clear(QUEUE_FLAG_DISCARD, rq);
+				queue_flag_clear(QUEUE_FLAG_SECDISCARD, rq);
 			}
 			__blk_end_request_all(req, error);
 			break;
 		case BLKIF_OP_FLUSH_DISKCACHE:
 		case BLKIF_OP_WRITE_BARRIER:
 			if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
 				printk(KERN_WARNING "blkfront: %s: write %s op failed\n",
 				       info->flush_op == BLKIF_OP_WRITE_BARRIER ?
 				       "barrier" :  "flush disk cache",
 				       info->gd->disk_name);
 				error = -EOPNOTSUPP;
 			}
 			if (unlikely(bret->status == BLKIF_RSP_ERROR &&
-				     info->shadow[id].req.nr_segments == 0)) {
+				     info->shadow[id].req.u.rw.nr_segments == 0)) {
 				printk(KERN_WARNING "blkfront: %s: empty write %s op failed\n",
 				       info->flush_op == BLKIF_OP_WRITE_BARRIER ?
 				       "barrier" :  "flush disk cache",
 				       info->gd->disk_name);
 				error = -EOPNOTSUPP;
 			}
 			if (unlikely(error)) {
 				if (error == -EOPNOTSUPP)
 					error = 0;
 				info->feature_flush = 0;
 				info->flush_op = 0;
 				xlvbd_flush(info);
 			}
 			/* fall through */
 		case BLKIF_OP_READ:
 		case BLKIF_OP_WRITE:
 			if (unlikely(bret->status != BLKIF_RSP_OKAY))
 				dev_dbg(&info->xbdev->dev, "Bad return from blkdev data "
 					"request: %x\n", bret->status);
 			__blk_end_request_all(req, error);
 			break;
 		default:
 			BUG();
 		}
 	}
 	info->ring.rsp_cons = i;
 	if (i != info->ring.req_prod_pvt) {
 		int more_to_do;
 		RING_FINAL_CHECK_FOR_RESPONSES(&info->ring, more_to_do);
 		if (more_to_do)
 			goto again;
 	} else
 		info->ring.sring->rsp_event = i + 1;
 	kick_pending_request_queues(info);
 	spin_unlock_irqrestore(&blkif_io_lock, flags);
 	return IRQ_HANDLED;
 }
 static int setup_blkring(struct xenbus_device *dev,
 			 struct blkfront_info *info)
 {
 	struct blkif_sring *sring;
 	int err;
 	info->ring_ref = GRANT_INVALID_REF;
 	sring = (struct blkif_sring *)__get_free_page(GFP_NOIO | __GFP_HIGH);
 	if (!sring) {
 		xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring");
 		return -ENOMEM;
 	}
 	SHARED_RING_INIT(sring);
 	FRONT_RING_INIT(&info->ring, sring, PAGE_SIZE);
 	sg_init_table(info->sg, BLKIF_MAX_SEGMENTS_PER_REQUEST);
 	err = xenbus_grant_ring(dev, virt_to_mfn(info->ring.sring));
 	if (err < 0) {
 		free_page((unsigned long)sring);
 		info->ring.sring = NULL;
 		goto fail;
 	}
 	info->ring_ref = err;
 	err = xenbus_alloc_evtchn(dev, &info->evtchn);
 	if (err)
 		goto fail;
 	err = bind_evtchn_to_irqhandler(info->evtchn,
 					blkif_interrupt,
 					IRQF_SAMPLE_RANDOM, "blkif", info);
 	if (err <= 0) {
 		xenbus_dev_fatal(dev, err,
 				 "bind_evtchn_to_irqhandler failed");
 		goto fail;
 	}
 	info->irq = err;
 	return 0;
 fail:
 	blkif_free(info, 0);
 	return err;
 }
 /* Common code used when first setting up, and when resuming. */
 static int talk_to_blkback(struct xenbus_device *dev,
 			   struct blkfront_info *info)
 {
 	const char *message = NULL;
 	struct xenbus_transaction xbt;
 	int err;
 	/* Create shared ring, alloc event channel. */
 	err = setup_blkring(dev, info);
 	if (err)
 		goto out;
 again:
 	err = xenbus_transaction_start(&xbt);
 	if (err) {
 		xenbus_dev_fatal(dev, err, "starting transaction");
 		goto destroy_blkring;
 	}
 	err = xenbus_printf(xbt, dev->nodename,
 			    "ring-ref", "%u", info->ring_ref);
 	if (err) {
 		message = "writing ring-ref";
 		goto abort_transaction;
 	}
 	err = xenbus_printf(xbt, dev->nodename,
 			    "event-channel", "%u", info->evtchn);
 	if (err) {
 		message = "writing event-channel";
 		goto abort_transaction;
 	}
 	err = xenbus_printf(xbt, dev->nodename, "protocol", "%s",
 			    XEN_IO_PROTO_ABI_NATIVE);
 	if (err) {
 		message = "writing protocol";
 		goto abort_transaction;
 	}
 	err = xenbus_transaction_end(xbt, 0);
 	if (err) {
 		if (err == -EAGAIN)
 			goto again;
 		xenbus_dev_fatal(dev, err, "completing transaction");
 		goto destroy_blkring;
 	}
 	xenbus_switch_state(dev, XenbusStateInitialised);
 	return 0;
  abort_transaction:
 	xenbus_transaction_end(xbt, 1);
 	if (message)
 		xenbus_dev_fatal(dev, err, "%s", message);
  destroy_blkring:
 	blkif_free(info, 0);
  out:
 	return err;
 }
 /**
  * Entry point to this code when a new device is created.  Allocate the basic
  * structures and the ring buffer for communication with the backend, and
  * inform the backend of the appropriate details for those.  Switch to
  * Initialised state.
  */
 static int blkfront_probe(struct xenbus_device *dev,
 			  const struct xenbus_device_id *id)
 {
 	int err, vdevice, i;
 	struct blkfront_info *info;
 	/* FIXME: Use dynamic device id if this is not set. */
 	err = xenbus_scanf(XBT_NIL, dev->nodename,
 			   "virtual-device", "%i", &vdevice);
 	if (err != 1) {
 		/* go looking in the extended area instead */
 		err = xenbus_scanf(XBT_NIL, dev->nodename, "virtual-device-ext",
 				   "%i", &vdevice);
 		if (err != 1) {
 			xenbus_dev_fatal(dev, err, "reading virtual-device");
 			return err;
 		}
 	}
 	if (xen_hvm_domain()) {
 		char *type;
 		int len;
 		/* no unplug has been done: do not hook devices != xen vbds */
 		if (xen_platform_pci_unplug & XEN_UNPLUG_UNNECESSARY) {
 			int major;
 			if (!VDEV_IS_EXTENDED(vdevice))
 				major = BLKIF_MAJOR(vdevice);
 			else
 				major = XENVBD_MAJOR;
 			if (major != XENVBD_MAJOR) {
 				printk(KERN_INFO
 						"%s: HVM does not support vbd %d as xen block device\n",
 						__FUNCTION__, vdevice);
 				return -ENODEV;
 			}
 		}
 		/* do not create a PV cdrom device if we are an HVM guest */
 		type = xenbus_read(XBT_NIL, dev->nodename, "device-type", &len);
 		if (IS_ERR(type))
 			return -ENODEV;
 		if (strncmp(type, "cdrom", 5) == 0) {
 			kfree(type);
 			return -ENODEV;
 		}
 		kfree(type);
 	}
 	info = kzalloc(sizeof(*info), GFP_KERNEL);
 	if (!info) {
 		xenbus_dev_fatal(dev, -ENOMEM, "allocating info structure");
 		return -ENOMEM;
 	}
 	mutex_init(&info->mutex);
 	info->xbdev = dev;
 	info->vdevice = vdevice;
 	info->connected = BLKIF_STATE_DISCONNECTED;
 	INIT_WORK(&info->work, blkif_restart_queue);
 	for (i = 0; i < BLK_RING_SIZE; i++)
-		info->shadow[i].req.id = i+1;
+		info->shadow[i].req.u.rw.id = i+1;
-	info->shadow[BLK_RING_SIZE-1].req.id = 0x0fffffff;
+	info->shadow[BLK_RING_SIZE-1].req.u.rw.id = 0x0fffffff;
 	/* Front end dir is a number, which is used as the id. */
 	info->handle = simple_strtoul(strrchr(dev->nodename, '/')+1, NULL, 0);
 	dev_set_drvdata(&dev->dev, info);
 	err = talk_to_blkback(dev, info);
 	if (err) {
 		kfree(info);
 		dev_set_drvdata(&dev->dev, NULL);
 		return err;
 	}
 	return 0;
 }
 static int blkif_recover(struct blkfront_info *info)
 {
 	int i;
 	struct blkif_request *req;
 	struct blk_shadow *copy;
 	int j;
 	/* Stage 1: Make a safe copy of the shadow state. */
 	copy = kmalloc(sizeof(info->shadow),
 		       GFP_NOIO | __GFP_REPEAT | __GFP_HIGH);
 	if (!copy)
 		return -ENOMEM;
 	memcpy(copy, info->shadow, sizeof(info->shadow));
 	/* Stage 2: Set up free list. */
 	memset(&info->shadow, 0, sizeof(info->shadow));
 	for (i = 0; i < BLK_RING_SIZE; i++)
-		info->shadow[i].req.id = i+1;
+		info->shadow[i].req.u.rw.id = i+1;
 	info->shadow_free = info->ring.req_prod_pvt;
-	info->shadow[BLK_RING_SIZE-1].req.id = 0x0fffffff;
+	info->shadow[BLK_RING_SIZE-1].req.u.rw.id = 0x0fffffff;
 	/* Stage 3: Find pending requests and requeue them. */
 	for (i = 0; i < BLK_RING_SIZE; i++) {
 		/* Not in use? */
 		if (!copy[i].request)
 			continue;
 		/* Grab a request slot and copy shadow state into it. */
 		req = RING_GET_REQUEST(&info->ring, info->ring.req_prod_pvt);
 		*req = copy[i].req;
 		/* We get a new request id, and must reset the shadow state. */
-		req->id = get_id_from_freelist(info);
+		req->u.rw.id = get_id_from_freelist(info);
-		memcpy(&info->shadow[req->id], &copy[i], sizeof(copy[i]));
+		memcpy(&info->shadow[req->u.rw.id], &copy[i], sizeof(copy[i]));
+		if (req->operation != BLKIF_OP_DISCARD) {
 		/* Rewrite any grant references invalidated by susp/resume. */
-		for (j = 0; j < req->nr_segments; j++)
+			for (j = 0; j < req->u.rw.nr_segments; j++)
-			gnttab_grant_foreign_access_ref(
+				gnttab_grant_foreign_access_ref(
-				req->u.rw.seg[j].gref,
+					req->u.rw.seg[j].gref,
-				info->xbdev->otherend_id,
+					info->xbdev->otherend_id,
-				pfn_to_mfn(info->shadow[req->id].frame[j]),
+					pfn_to_mfn(info->shadow[req->u.rw.id].frame[j]),
-				rq_data_dir(info->shadow[req->id].request));
+					rq_data_dir(info->shadow[req->u.rw.id].request));
-		info->shadow[req->id].req = *req;
+		}
+		info->shadow[req->u.rw.id].req = *req;
 		info->ring.req_prod_pvt++;
 	}
 	kfree(copy);
 	xenbus_switch_state(info->xbdev, XenbusStateConnected);
 	spin_lock_irq(&blkif_io_lock);
 	/* Now safe for us to use the shared ring */
 	info->connected = BLKIF_STATE_CONNECTED;
 	/* Send off requeued requests */
 	flush_requests(info);
 	/* Kick any other new requests queued since we resumed */
 	kick_pending_request_queues(info);
 	spin_unlock_irq(&blkif_io_lock);
 	return 0;
 }
 /**
  * We are reconnecting to the backend, due to a suspend/resume, or a backend
  * driver restart.  We tear down our blkif structure and recreate it, but
  * leave the device-layer structures intact so that this is transparent to the
  * rest of the kernel.
  */
 static int blkfront_resume(struct xenbus_device *dev)
 {
 	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
 	int err;
 	dev_dbg(&dev->dev, "blkfront_resume: %s\n", dev->nodename);
 	blkif_free(info, info->connected == BLKIF_STATE_CONNECTED);
 	err = talk_to_blkback(dev, info);
 	if (info->connected == BLKIF_STATE_SUSPENDED && !err)
 		err = blkif_recover(info);
 	return err;
 }
 static void
 blkfront_closing(struct blkfront_info *info)
 {
 	struct xenbus_device *xbdev = info->xbdev;
 	struct block_device *bdev = NULL;
 	mutex_lock(&info->mutex);
 	if (xbdev->state == XenbusStateClosing) {
 		mutex_unlock(&info->mutex);
 		return;
 	}
 	if (info->gd)
 		bdev = bdget_disk(info->gd, 0);
 	mutex_unlock(&info->mutex);
 	if (!bdev) {
 		xenbus_frontend_closed(xbdev);
 		return;
 	}
 	mutex_lock(&bdev->bd_mutex);
 	if (bdev->bd_openers) {
 		xenbus_dev_error(xbdev, -EBUSY,
 				 "Device in use; refusing to close");
 		xenbus_switch_state(xbdev, XenbusStateClosing);
 	} else {
 		xlvbd_release_gendisk(info);
 		xenbus_frontend_closed(xbdev);
 	}
 	mutex_unlock(&bdev->bd_mutex);
 	bdput(bdev);
 }
 static void blkfront_setup_discard(struct blkfront_info *info)
 {
 	int err;
 	char *type;
 	unsigned int discard_granularity;
 	unsigned int discard_alignment;
+	unsigned int discard_secure;
 	type = xenbus_read(XBT_NIL, info->xbdev->otherend, "type", NULL);
 	if (IS_ERR(type))
 		return;
+	info->feature_secdiscard = 0;
 	if (strncmp(type, "phy", 3) == 0) {
 		err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
 			"discard-granularity", "%u", &discard_granularity,
 			"discard-alignment", "%u", &discard_alignment,
 			NULL);
 		if (!err) {
 			info->feature_discard = 1;
 			info->discard_granularity = discard_granularity;
 			info->discard_alignment = discard_alignment;
 		}
+		err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
+			    "discard-secure", "%d", &discard_secure,
+			    NULL);
+		if (!err)
+			info->feature_secdiscard = discard_secure;
 	} else if (strncmp(type, "file", 4) == 0)
 		info->feature_discard = 1;
 	kfree(type);
 }
 /*
  * Invoked when the backend is finally 'ready' (and has told produced
  * the details about the physical device - #sectors, size, etc).
  */
 static void blkfront_connect(struct blkfront_info *info)
 {
 	unsigned long long sectors;
 	unsigned long sector_size;
 	unsigned int binfo;
 	int err;
 	int barrier, flush, discard;
 	switch (info->connected) {
 	case BLKIF_STATE_CONNECTED:
 		/*
 		 * Potentially, the back-end may be signalling
 		 * a capacity change; update the capacity.
 		 */
 		err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
 				   "sectors", "%Lu", &sectors);
 		if (XENBUS_EXIST_ERR(err))
 			return;
 		printk(KERN_INFO "Setting capacity to %Lu\n",
 		       sectors);
 		set_capacity(info->gd, sectors);
 		revalidate_disk(info->gd);
 		/* fall through */
 	case BLKIF_STATE_SUSPENDED:
 		return;
 	default:
 		break;
 	}
 	dev_dbg(&info->xbdev->dev, "%s:%s.\n",
 		__func__, info->xbdev->otherend);
 	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
 			    "sectors", "%llu", &sectors,
 			    "info", "%u", &binfo,
 			    "sector-size", "%lu", &sector_size,
 			    NULL);
 	if (err) {
 		xenbus_dev_fatal(info->xbdev, err,
 				 "reading backend fields at %s",
 				 info->xbdev->otherend);
 		return;
 	}
 	info->feature_flush = 0;
 	info->flush_op = 0;
 	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
 			    "feature-barrier", "%d", &barrier,
 			    NULL);
 	/*
 	 * If there's no "feature-barrier" defined, then it means
 	 * we're dealing with a very old backend which writes
 	 * synchronously; nothing to do.
 	 *
 	 * If there are barriers, then we use flush.
 	 */
 	if (!err && barrier) {
 		info->feature_flush = REQ_FLUSH | REQ_FUA;
 		info->flush_op = BLKIF_OP_WRITE_BARRIER;
 	}
 	/*
 	 * And if there is "feature-flush-cache" use that above
 	 * barriers.
 	 */
 	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
 			    "feature-flush-cache", "%d", &flush,
 			    NULL);
 	if (!err && flush) {
 		info->feature_flush = REQ_FLUSH;
 		info->flush_op = BLKIF_OP_FLUSH_DISKCACHE;
 	}
 	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
 			    "feature-discard", "%d", &discard,
 			    NULL);
 	if (!err && discard)
 		blkfront_setup_discard(info);
 	err = xlvbd_alloc_gendisk(sectors, info, binfo, sector_size);
 	if (err) {
 		xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s",
 				 info->xbdev->otherend);
 		return;
 	}
 	xenbus_switch_state(info->xbdev, XenbusStateConnected);
 	/* Kick pending requests. */
 	spin_lock_irq(&blkif_io_lock);
 	info->connected = BLKIF_STATE_CONNECTED;
 	kick_pending_request_queues(info);
 	spin_unlock_irq(&blkif_io_lock);
 	add_disk(info->gd);
 	info->is_ready = 1;
 }
 /**
  * Callback received when the backend's state changes.
  */
 static void blkback_changed(struct xenbus_device *dev,
 			    enum xenbus_state backend_state)
 {
 	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
 	dev_dbg(&dev->dev, "blkfront:blkback_changed to state %d.\n", backend_state);
 	switch (backend_state) {
 	case XenbusStateInitialising:
 	case XenbusStateInitWait:
 	case XenbusStateInitialised:
 	case XenbusStateReconfiguring:
 	case XenbusStateReconfigured:
 	case XenbusStateUnknown:
 	case XenbusStateClosed:
 		break;
 	case XenbusStateConnected:
 		blkfront_connect(info);
 		break;
 	case XenbusStateClosing:
 		blkfront_closing(info);
 		break;
 	}
 }
 static int blkfront_remove(struct xenbus_device *xbdev)
 {
 	struct blkfront_info *info = dev_get_drvdata(&xbdev->dev);
 	struct block_device *bdev = NULL;
 	struct gendisk *disk;
 	dev_dbg(&xbdev->dev, "%s removed", xbdev->nodename);
 	blkif_free(info, 0);
 	mutex_lock(&info->mutex);
 	disk = info->gd;
 	if (disk)
 		bdev = bdget_disk(disk, 0);
 	info->xbdev = NULL;
 	mutex_unlock(&info->mutex);
 	if (!bdev) {
 		kfree(info);
 		return 0;
 	}
 	/*
 	 * The xbdev was removed before we reached the Closed
 	 * state. See if it's safe to remove the disk. If the bdev
 	 * isn't closed yet, we let release take care of it.
 	 */
 	mutex_lock(&bdev->bd_mutex);
 	info = disk->private_data;
 	dev_warn(disk_to_dev(disk),
 		 "%s was hot-unplugged, %d stale handles\n",
 		 xbdev->nodename, bdev->bd_openers);
 	if (info && !bdev->bd_openers) {
 		xlvbd_release_gendisk(info);
 		disk->private_data = NULL;
 		kfree(info);
 	}
 	mutex_unlock(&bdev->bd_mutex);
 	bdput(bdev);
 	return 0;
 }
 static int blkfront_is_ready(struct xenbus_device *dev)
 {
 	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
 	return info->is_ready && info->xbdev;
 }
 static int blkif_open(struct block_device *bdev, fmode_t mode)
 {
 	struct gendisk *disk = bdev->bd_disk;
 	struct blkfront_info *info;
 	int err = 0;
 	mutex_lock(&blkfront_mutex);
 	info = disk->private_data;
 	if (!info) {
 		/* xbdev gone */
 		err = -ERESTARTSYS;
 		goto out;
 	}
 	mutex_lock(&info->mutex);
 	if (!info->gd)
 		/* xbdev is closed */
 		err = -ERESTARTSYS;
 	mutex_unlock(&info->mutex);
 out:
 	mutex_unlock(&blkfront_mutex);
 	return err;
 }
 static int blkif_release(struct gendisk *disk, fmode_t mode)
 {
 	struct blkfront_info *info = disk->private_data;
 	struct block_device *bdev;
 	struct xenbus_device *xbdev;
 	mutex_lock(&blkfront_mutex);
 	bdev = bdget_disk(disk, 0);
 	bdput(bdev);
 	if (bdev->bd_openers)
 		goto out;
 	/*
 	 * Check if we have been instructed to close. We will have
 	 * deferred this request, because the bdev was still open.
 	 */
 	mutex_lock(&info->mutex);
 	xbdev = info->xbdev;
 	if (xbdev && xbdev->state == XenbusStateClosing) {
 		/* pending switch to state closed */
 		dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n");
 		xlvbd_release_gendisk(info);
 		xenbus_frontend_closed(info->xbdev);
  	}
 	mutex_unlock(&info->mutex);
 	if (!xbdev) {
 		/* sudden device removal */
 		dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n");
 		xlvbd_release_gendisk(info);
 		disk->private_data = NULL;
 		kfree(info);
 	}
 out:
 	mutex_unlock(&blkfront_mutex);
 	return 0;
 }
 static const struct block_device_operations xlvbd_block_fops =
 {
 	.owner = THIS_MODULE,
 	.open = blkif_open,
 	.release = blkif_release,
 	.getgeo = blkif_getgeo,
 	.ioctl = blkif_ioctl,
 };
 static const struct xenbus_device_id blkfront_ids[] = {
 	{ "vbd" },
 	{ "" }
 };
 static DEFINE_XENBUS_DRIVER(blkfront, ,
 	.probe = blkfront_probe,
 	.remove = blkfront_remove,
 	.resume = blkfront_resume,
 	.otherend_changed = blkback_changed,
 	.is_ready = blkfront_is_ready,
 );
 static int __init xlblk_init(void)
 {
 	int ret;
 	if (!xen_domain())
 		return -ENODEV;
 	if (register_blkdev(XENVBD_MAJOR, DEV_NAME)) {
 		printk(KERN_WARNING "xen_blk: can't get major %d with name %s\n",
 		       XENVBD_MAJOR, DEV_NAME);
 		return -ENODEV;
 	}
 	ret = xenbus_register_frontend(&blkfront_driver);
 	if (ret) {
 		unregister_blkdev(XENVBD_MAJOR, DEV_NAME);
 		return ret;
 	}
 	return 0;
 }
 module_init(xlblk_init);
 static void __exit xlblk_exit(void)
 {
 	return xenbus_unregister_driver(&blkfront_driver);
 }
 module_exit(xlblk_exit);
 MODULE_DESCRIPTION("Xen virtual block device frontend");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS_BLOCKDEV_MAJOR(XENVBD_MAJOR);
 MODULE_ALIAS("xen:vbd");
 MODULE_ALIAS("xenblk");

include/xen/interface/io/blkif.h

Diff comments View file @ 16008d6

 /******************************************************************************
  * blkif.h
  *
  * Unified block-device I/O interface for Xen guest OSes.
  *
  * Copyright (c) 2003-2004, Keir Fraser
  */
 #ifndef __XEN_PUBLIC_IO_BLKIF_H__
 #define __XEN_PUBLIC_IO_BLKIF_H__
 #include "ring.h"
 #include "../grant_table.h"
 /*
  * Front->back notifications: When enqueuing a new request, sending a
  * notification can be made conditional on req_event (i.e., the generic
  * hold-off mechanism provided by the ring macros). Backends must set
  * req_event appropriately (e.g., using RING_FINAL_CHECK_FOR_REQUESTS()).
  *
  * Back->front notifications: When enqueuing a new response, sending a
  * notification can be made conditional on rsp_event (i.e., the generic
  * hold-off mechanism provided by the ring macros). Frontends must set
  * rsp_event appropriately (e.g., using RING_FINAL_CHECK_FOR_RESPONSES()).
  */
 typedef uint16_t blkif_vdev_t;
 typedef uint64_t blkif_sector_t;
 /*
  * REQUEST CODES.
  */
 #define BLKIF_OP_READ              0
 #define BLKIF_OP_WRITE             1
 /*
  * Recognised only if "feature-barrier" is present in backend xenbus info.
  * The "feature_barrier" node contains a boolean indicating whether barrier
  * requests are likely to succeed or fail. Either way, a barrier request
  * may fail at any time with BLKIF_RSP_EOPNOTSUPP if it is unsupported by
  * the underlying block-device hardware. The boolean simply indicates whether
  * or not it is worthwhile for the frontend to attempt barrier requests.
  * If a backend does not recognise BLKIF_OP_WRITE_BARRIER, it should *not*
  * create the "feature-barrier" node!
  */
 #define BLKIF_OP_WRITE_BARRIER     2
 /*
  * Recognised if "feature-flush-cache" is present in backend xenbus
  * info.  A flush will ask the underlying storage hardware to flush its
  * non-volatile caches as appropriate.  The "feature-flush-cache" node
  * contains a boolean indicating whether flush requests are likely to
  * succeed or fail. Either way, a flush request may fail at any time
  * with BLKIF_RSP_EOPNOTSUPP if it is unsupported by the underlying
  * block-device hardware. The boolean simply indicates whether or not it
  * is worthwhile for the frontend to attempt flushes.  If a backend does
  * not recognise BLKIF_OP_WRITE_FLUSH_CACHE, it should *not* create the
  * "feature-flush-cache" node!
  */
 #define BLKIF_OP_FLUSH_DISKCACHE   3
 /*
  * Recognised only if "feature-discard" is present in backend xenbus info.
  * The "feature-discard" node contains a boolean indicating whether trim
  * (ATA) or unmap (SCSI) - conviently called discard requests are likely
  * to succeed or fail. Either way, a discard request
  * may fail at any time with BLKIF_RSP_EOPNOTSUPP if it is unsupported by
  * the underlying block-device hardware. The boolean simply indicates whether
  * or not it is worthwhile for the frontend to attempt discard requests.
  * If a backend does not recognise BLKIF_OP_DISCARD, it should *not*
  * create the "feature-discard" node!
  *
  * Discard operation is a request for the underlying block device to mark
  * extents to be erased. However, discard does not guarantee that the blocks
  * will be erased from the device - it is just a hint to the device
  * controller that these blocks are no longer in use. What the device
  * controller does with that information is left to the controller.
  * Discard operations are passed with sector_number as the
  * sector index to begin discard operations at and nr_sectors as the number of
  * sectors to be discarded. The specified sectors should be discarded if the
  * underlying block device supports trim (ATA) or unmap (SCSI) operations,
  * or a BLKIF_RSP_EOPNOTSUPP  should be returned.
  * More information about trim/unmap operations at:
  * http://t13.org/Documents/UploadedDocuments/docs2008/
  *     e07154r6-Data_Set_Management_Proposal_for_ATA-ACS2.doc
  * http://www.seagate.com/staticfiles/support/disc/manuals/
  *     Interface%20manuals/100293068c.pdf
+ * The backend can optionally provide three extra XenBus attributes to
+ * further optimize the discard functionality:
+ * 'discard-aligment' - Devices that support discard functionality may
+ * internally allocate space in units that are bigger than the exported
+ * logical block size. The discard-alignment parameter indicates how many bytes
+ * the beginning of the partition is offset from the internal allocation unit's
+ * natural alignment.
+ * 'discard-granularity'  - Devices that support discard functionality may
+ * internally allocate space using units that are bigger than the logical block
+ * size. The discard-granularity parameter indicates the size of the internal
+ * allocation unit in bytes if reported by the device. Otherwise the
+ * discard-granularity will be set to match the device's physical block size.
+ * 'discard-secure' - All copies of the discarded sectors (potentially created
+ * by garbage collection) must also be erased.  To use this feature, the flag
+ * BLKIF_DISCARD_SECURE must be set in the blkif_request_trim.
  */
 #define BLKIF_OP_DISCARD           5
 /*
  * Maximum scatter/gather segments per request.
  * This is carefully chosen so that sizeof(struct blkif_ring) <= PAGE_SIZE.
  * NB. This could be 12 if the ring indexes weren't stored in the same page.
  */
 #define BLKIF_MAX_SEGMENTS_PER_REQUEST 11
 struct blkif_request_rw {
+	uint8_t        nr_segments;  /* number of segments                   */
+	blkif_vdev_t   handle;       /* only for read/write requests         */
+#ifdef CONFIG_X86_64
+	uint32_t       _pad1;	     /* offsetof(blkif_request,u.rw.id) == 8 */
+#endif
+	uint64_t       id;           /* private guest value, echoed in resp  */
 	blkif_sector_t sector_number;/* start sector idx on disk (r/w only)  */
 	struct blkif_request_segment {
 		grant_ref_t gref;        /* reference to I/O buffer frame        */
 		/* @first_sect: first sector in frame to transfer (inclusive).   */
 		/* @last_sect: last sector in frame to transfer (inclusive).     */
 		uint8_t     first_sect, last_sect;
 	} seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
-};
+} __attribute__((__packed__));
 struct blkif_request_discard {
+	uint8_t        flag;         /* BLKIF_DISCARD_SECURE or zero.        */
+#define BLKIF_DISCARD_SECURE (1<<0)  /* ignored if discard-secure=0          */
+	blkif_vdev_t   _pad1;        /* only for read/write requests         */
+#ifdef CONFIG_X86_64
+	uint32_t       _pad2;        /* offsetof(blkif_req..,u.discard.id)==8*/
+#endif
+	uint64_t       id;           /* private guest value, echoed in resp  */
 	blkif_sector_t sector_number;
-	uint64_t nr_sectors;
+	uint64_t       nr_sectors;
-};
+	uint8_t        _pad3;
+} __attribute__((__packed__));
 struct blkif_request {
 	uint8_t        operation;    /* BLKIF_OP_???                         */
-	uint8_t        nr_segments;  /* number of segments                   */
-	blkif_vdev_t   handle;       /* only for read/write requests         */
-	uint64_t       id;           /* private guest value, echoed in resp  */
 	union {
 		struct blkif_request_rw rw;
 		struct blkif_request_discard discard;
 	} u;
-};
+} __attribute__((__packed__));
 struct blkif_response {
 	uint64_t        id;              /* copied from request */
 	uint8_t         operation;       /* copied from request */
 	int16_t         status;          /* BLKIF_RSP_???       */
 };
 /*
  * STATUS RETURN CODES.
  */
  /* Operation not supported (only happens on barrier writes). */
 #define BLKIF_RSP_EOPNOTSUPP  -2
  /* Operation failed for some unspecified reason (-EIO). */
 #define BLKIF_RSP_ERROR       -1
  /* Operation completed successfully. */
 #define BLKIF_RSP_OKAY         0
 /*
  * Generate blkif ring structures and types.
  */
 DEFINE_RING_TYPES(blkif, struct blkif_request, struct blkif_response);
 #define VDISK_CDROM        0x1
 #define VDISK_REMOVABLE    0x2
 #define VDISK_READONLY     0x4
 /* Xen-defined major numbers for virtual disks, they look strangely
  * familiar */
 #define XEN_IDE0_MAJOR	3
 #define XEN_IDE1_MAJOR	22
 #define XEN_SCSI_DISK0_MAJOR	8
 #define XEN_SCSI_DISK1_MAJOR	65
 #define XEN_SCSI_DISK2_MAJOR	66
 #define XEN_SCSI_DISK3_MAJOR	67
 #define XEN_SCSI_DISK4_MAJOR	68
 #define XEN_SCSI_DISK5_MAJOR	69
 #define XEN_SCSI_DISK6_MAJOR	70
 #define XEN_SCSI_DISK7_MAJOR	71
 #define XEN_SCSI_DISK8_MAJOR	128
 #define XEN_SCSI_DISK9_MAJOR	129
 #define XEN_SCSI_DISK10_MAJOR	130
 #define XEN_SCSI_DISK11_MAJOR	131
 #define XEN_SCSI_DISK12_MAJOR	132
 #define XEN_SCSI_DISK13_MAJOR	133
 #define XEN_SCSI_DISK14_MAJOR	134
 #define XEN_SCSI_DISK15_MAJOR	135