Menu
▾
▴
[Scst-svn] SF.net SVN: scst: [304] trunk
|
From: <vl...@us...> - 2008-03-17 11:29:52
|
Revision: 304
http://scst.svn.sourceforge.net/scst/?rev=304&view=rev
Author: vlnb
Date: 2008-03-17 04:29:50 -0700 (Mon, 17 Mar 2008)
Log Message:
-----------
Patch from Vu Pham <vu...@me...>: SRP target driver
Modified Paths:
--------------
trunk/Makefile
Added Paths:
-----------
trunk/srpt/
trunk/srpt/LICENSE
trunk/srpt/Makefile
trunk/srpt/README
trunk/srpt/README.ofed
trunk/srpt/patches/
trunk/srpt/patches/scst_increase_max_tgt_cmds.patch
trunk/srpt/src/
trunk/srpt/src/Kconfig
trunk/srpt/src/Makefile
trunk/srpt/src/Makefile.in_kernel
trunk/srpt/src/ib_dm_mad.h
trunk/srpt/src/ib_srpt.c
trunk/srpt/src/ib_srpt.h
Modified: trunk/Makefile
===================================================================
--- trunk/Makefile 2008-03-14 10:42:03 UTC (rev 303)
+++ trunk/Makefile 2008-03-17 11:29:50 UTC (rev 304)
@@ -23,6 +23,7 @@
QLA_DIR=qla2x00t/qla2x00-target
LSI_DIR=mpt
USR_DIR=usr/fileio
+SRP_DIR=srpt
ISCSI_DIR=iscsi-scst
#ISCSI_DISTDIR=../../../iscsi_scst_inst
@@ -58,6 +59,12 @@
@echo " lsi_install : lsi target: install"
@echo " lsi_uninstall : lsi target: uninstall"
@echo ""
+ @echo " srpt : make SRP target"
+ @echo " srpt_clean : srp target: clean "
+ @echo " srpt_extraclean : srp target: clean + clean dependencies"
+ @echo " srpt_install : srp target: install"
+ @echo " srpt_uninstall : srp target: uninstall"
+ @echo ""
@echo " usr : make user space fileio_tgt target"
@echo " usr_clean : usr target: clean "
@echo " usr_extraclean : usr target: clean + clean dependencies"
@@ -76,6 +83,7 @@
cd $(SCST_DIR) && $(MAKE) $@
@if [ -d $(QLA_DIR) ]; then cd $(QLA_DIR) && $(MAKE) $@; fi
# @if [ -d $(LSI_DIR) ]; then cd $(LSI_DIR) && $(MAKE) $@; fi
+ @if [ -d $(SRP_DIR) ]; then cd $(SRP_DIR) && $(MAKE) $@; fi
@if [ -d $(ISCSI_DIR) ]; then cd $(ISCSI_DIR) && $(MAKE) $@; fi
@if [ -d $(USR_DIR) ]; then cd $(USR_DIR) && $(MAKE) $@; fi
@@ -83,6 +91,7 @@
cd $(SCST_DIR) && $(MAKE) $@
@if [ -d $(QLA_DIR) ]; then cd $(QLA_DIR) && $(MAKE) $@; fi
# @if [ -d $(LSI_DIR) ]; then cd $(LSI_DIR) && $(MAKE) $@; fi
+ @if [ -d $(SRP_DIR) ]; then cd $(SRP_DIR) && $(MAKE) $@; fi
@if [ -d $(ISCSI_DIR) ]; then cd $(ISCSI_DIR) && $(MAKE) DISTDIR=$(ISCSI_DISTDIR) $@; fi
@if [ -d $(USR_DIR) ]; then cd $(USR_DIR) && $(MAKE) $@; fi
@@ -90,6 +99,7 @@
cd $(SCST_DIR) && $(MAKE) $@
@if [ -d $(QLA_DIR) ]; then cd $(QLA_DIR) && $(MAKE) $@; fi
@if [ -d $(LSI_DIR) ]; then cd $(LSI_DIR) && $(MAKE) $@; fi
+ @if [ -d $(SRP_DIR) ]; then cd $(SRP_DIR) && $(MAKE) $@; fi
@if [ -d $(ISCSI_DIR) ]; then cd $(ISCSI_DIR) && $(MAKE) $@; fi
@if [ -d $(USR_DIR) ]; then cd $(USR_DIR) && $(MAKE) $@; fi
@@ -98,6 +108,7 @@
@if [ -d $(QLA_INI_DIR) ]; then cd $(QLA_INI_DIR) && $(MAKE) $@; fi
@if [ -d $(QLA_DIR) ]; then cd $(QLA_DIR) && $(MAKE) $@; fi
@if [ -d $(LSI_DIR) ]; then cd $(LSI_DIR) && $(MAKE) $@; fi
+ @if [ -d $(SRP_DIR) ]; then cd $(SRP_DIR) && $(MAKE) $@; fi
@if [ -d $(ISCSI_DIR) ]; then cd $(ISCSI_DIR) && $(MAKE) $@; fi
@if [ -d $(USR_DIR) ]; then cd $(USR_DIR) && $(MAKE) $@; fi
@@ -106,6 +117,7 @@
@if [ -d $(QLA_INI_DIR) ]; then cd $(QLA_INI_DIR) && $(MAKE) $@; fi
@if [ -d $(QLA_DIR) ]; then cd $(QLA_DIR) && $(MAKE) $@; fi
@if [ -d $(LSI_DIR) ]; then cd $(LSI_DIR) && $(MAKE) $@; fi
+ @if [ -d $(SRP_DIR) ]; then cd $(SRP_DIR) && $(MAKE) $@; fi
@if [ -d $(ISCSI_DIR) ]; then cd $(ISCSI_DIR) && $(MAKE) $@; fi
@if [ -d $(USR_DIR) ]; then cd $(USR_DIR) && $(MAKE) $@; fi
@@ -171,6 +183,21 @@
lsi_extraclean:
cd $(LSI_DIR) && $(MAKE) extraclean
+srpt:
+ cd $(SRP_DIR) && $(MAKE)
+
+srpt_install:
+ cd $(SRP_DIR) && $(MAKE) install
+
+srpt_uninstall:
+ cd $(SRP_DIR) && $(MAKE) uninstall
+
+srpt_clean:
+ cd $(SRP_DIR) && $(MAKE) clean
+
+srpt_extraclean:
+ cd $(SRP_DIR) && $(MAKE) extraclean
+
usr:
cd $(USR_DIR) && $(MAKE)
Property changes on: trunk/srpt
___________________________________________________________________
Name: svn:ignore
+ *.o
*.mod.c
*.ko
*.ko.cmd
*.o.cmd
.tmp_versions
Modules.symvers
Module.symvers
Added: trunk/srpt/LICENSE
===================================================================
--- trunk/srpt/LICENSE (rev 0)
+++ trunk/srpt/LICENSE 2008-03-17 11:29:50 UTC (rev 304)
@@ -0,0 +1,45 @@
+OpenIB.org BSD license:
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+* Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following
+disclaimer in the documentation and/or other materials provided
+with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+
+Alternatively, this software may be distributed under the terms of the
+the GNU Public License ("GPL") with platforms where the prevalant license
+is the GNU Public License:
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of The Version 2 GNU General Public License as published
+ by the Free Software Foundation.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
Added: trunk/srpt/Makefile
===================================================================
--- trunk/srpt/Makefile (rev 0)
+++ trunk/srpt/Makefile 2008-03-17 11:29:50 UTC (rev 304)
@@ -0,0 +1,57 @@
+#
+# Makefile for the Linux kernel device drivers.
+#
+# Note! Dependencies are done automagically by 'make dep', which also
+# removes any old dependencies. DON'T put your own dependencies here
+# unless it's something special (not a .c file).
+#
+# Note 2! The CFLAGS definitions are now in the main makefile.
+
+SCST_DIR := $(shell pwd)/../scst/src
+SUBDIRS := $(shell pwd)
+
+ifeq ($(KVER),)
+ ifeq ($(KDIR),)
+ KVER = $(shell uname -r)
+ KDIR ?= /lib/modules/$(KVER)/build
+ else
+ KVER = $$KERNELRELEASE
+ endif
+else
+ KDIR ?= /lib/modules/$(KVER)/build
+endif
+
+all: Modules.symvers Module.symvers
+ $(MAKE) -C $(KDIR) SUBDIRS=$(shell pwd)/src modules
+
+install: all src/ib_srpt.ko
+ @eval `sed -n 's/#define UTS_RELEASE /KERNELRELEASE=/p' $(KDIR)/include/linux/version.h $(KDIR)/include/linux/utsrelease.h 2>/dev/null`; \
+ install -vD -m 644 src/ib_srpt.ko \
+ $(DISTDIR)$(INSTALL_MOD_PATH)/lib/modules/$(KVER)/extra/ib_srpt.ko
+ -/sbin/depmod -aq $(KVER)
+
+SCST_MOD_VERS := $(shell ls $(SCST_DIR)/Modules.symvers 2>/dev/null)
+ifneq ($(SCST_MOD_VERS),)
+Modules.symvers: $(SCST_DIR)/Modules.symvers
+ echo $(SCST_MOD_VERS)
+ cp $(SCST_DIR)/Modules.symvers src/
+else
+.PHONY: Modules.symvers
+endif
+
+# It's renamed in 2.6.18
+SCST_MOD_VERS := $(shell ls $(SCST_DIR)/Module.symvers 2>/dev/null)
+ifneq ($(SCST_MOD_VERS),)
+Module.symvers: $(SCST_DIR)/Module.symvers
+ cp $(SCST_DIR)/Module.symvers src/
+else
+.PHONY: Module.symvers
+endif
+
+clean:
+ $(MAKE) -C $(KDIR) SUBDIRS=$(shell pwd)/src clean
+ rm -f src/Modules.symvers src/Module.symvers
+
+extraclean: clean
+
+.PHONY: all install clean extraclean
Added: trunk/srpt/README
===================================================================
--- trunk/srpt/README (rev 0)
+++ trunk/srpt/README 2008-03-17 11:29:50 UTC (rev 304)
@@ -0,0 +1,106 @@
+SCSI RDMA Protocol (SRP) Target driver for Linux
+=================================================
+
+SRP Target driver is designed to work directly on top of OpenFabrics
+OFED-1.x software stack (http://www.openfabrics.org) or Infiniband
+drivers in Linux kernel tree (kernel.org). It also interfaces with
+Generic SCSI target mid-level driver - SCST (http://scst.sourceforge.net)
+
+NOTES: This SRP Target driver can only compile and work with IB driver
+ in Linux vanilla kernel. It does not compile and work with IB
+ driver in OFED-1.x packages
+
+ If you want to work with IB driver in OFED-1.x package, you should
+ read and follow instruction in README.ofed file
+
+
+Installation
+------------
+$ make
+$ make install
+
+To minimize QUEUEFULL conditions, please apply scst_increase_max_tgt_cmds
+patch and recompile scst
+
+$ cd ~scst/trunk
+$ patch -p0 < srpt/patches/scst_increasa_max_tgt_cmds.patch
+$ make scst scst_install srpt srpt_install
+
+
+How-to run
+-----------
+A. On srp target machine
+1. Please refer to SCST's README for loading scst driver and its
+dev_handlers drivers (scst_disk, scst_vdisk block or file IO mode, nullio, ...)
+
+Example 1: working with real back-end scsi disks
+a. modprobe scst
+b. modprobe scst_disk
+c. cat /proc/scsi_tgt/scsi_tgt
+
+ibstor00:~ # cat /proc/scsi_tgt/scsi_tgt
+Device (host:ch:id:lun or name) Device handler
+0:0:0:0 dev_disk
+4:0:0:0 dev_disk
+5:0:0:0 dev_disk
+6:0:0:0 dev_disk
+7:0:0:0 dev_disk
+
+Now you want to exclude the first scsi disk and expose the last 4 scsi disks as
+IB/SRP luns for I/O
+echo "add 4:0:0:0 0" >/proc/scsi_tgt/groups/Default/devices
+echo "add 5:0:0:0 1" >/proc/scsi_tgt/groups/Default/devices
+echo "add 6:0:0:0 2" >/proc/scsi_tgt/groups/Default/devices
+echo "add 7:0:0:0 3" >/proc/scsi_tgt/groups/Default/devices
+
+Example 2: working with VDISK FILEIO mode (using md0 device and file 10G-file)
+a. modprobe scst
+b. modprobe scst_vdisk
+c. echo "open vdisk0 /dev/md0" > /proc/scsi_tgt/vdisk/vdisk
+d. echo "open vdisk1 /10G-file" > /proc/scsi_tgt/vdisk/vdisk
+e. echo "add vdisk0 0" >/proc/scsi_tgt/groups/Default/devices
+f. echo "add vdisk1 1" >/proc/scsi_tgt/groups/Default/devices
+
+Example 3: working with VDISK BLOCKIO mode (using md0 device, sda, and cciss/c1d0)
+a. modprobe scst
+b. modprobe scst_vdisk
+c. echo "open vdisk0 /dev/md0 BLOCKIO" > /proc/scsi_tgt/vdisk/vdisk
+d. echo "open vdisk1 /dev/sda BLOCKIO" > /proc/scsi_tgt/vdisk/vdisk
+e. echo "open vdisk2 /dev/cciss/c1d0 BLOCKIO" > /proc/scsi_tgt/vdisk/vdisk
+f. echo "add vdisk0 0" >/proc/scsi_tgt/groups/Default/devices
+g. echo "add vdisk1 1" >/proc/scsi_tgt/groups/Default/devices
+h. echo "add vdisk2 2" >/proc/scsi_tgt/groups/Default/devices
+
+2. modprobe ib_srpt
+
+
+B. On initiator machines you can manualy do the following steps:
+1. modprobe ib_srp
+2. ipsrpdm -c (to discover new SRP target)
+3. echo <new target info> > /sys/class/infiniband_srp/srp-mthca0-1/add_target
+4. fdisk -l (will show new discovered scsi disks)
+
+Example:
+Assume that you use port 1 of first HCA in the system ie. mthca0
+
+[root@lab104 ~]# ibsrpdm -c -d /dev/infiniband/umad0
+id_ext=0002c90200226cf4,ioc_guid=0002c90200226cf4,
+dgid=fe800000000000000002c90200226cf5,pkey=ffff,service_id=0002c90200226cf4
+[root@lab104 ~]# echo id_ext=0002c90200226cf4,ioc_guid=0002c90200226cf4,
+dgid=fe800000000000000002c90200226cf5,pkey=ffff,service_id=0002c90200226cf4 >
+/sys/class/infiniband_srp/srp-mthca0-1/add_target
+
+OR
+
++ You can edit /etc/infiniband/openib.conf to load srp driver and srp HA daemon
+automatically ie. set SRP_LOAD=yes, and SRPHA_ENABLE=yes
++ To set up and use high availability feature you need dm-multipath driver
+and multipath tool
++ Please refer to OFED-1.x SRP's user manual for more in-details instructions
+on how-to enable/use HA feature
+
+
+TO DO
+------
++ Stress test and stabilize the code
++ Performance tuning
Added: trunk/srpt/README.ofed
===================================================================
--- trunk/srpt/README.ofed (rev 0)
+++ trunk/srpt/README.ofed 2008-03-17 11:29:50 UTC (rev 304)
@@ -0,0 +1,165 @@
+SCSI RDMA Protocol (SRP) Target driver for Linux
+=================================================
+
+SRP Target driver is designed to work directly on top of OpenFabrics
+OFED-1.x software stack (http://www.openfabrics.org) or Infiniband
+drivers in Linux kernel tree (kernel.org). It also interfaces with
+Generic SCSI target mid-level driver - SCST (http://scst.sourceforge.net)
+
+By interfacing with SCST driver we are able to work and support a lot IO
+modes on real or virtual devices in the backend
+1. scst_disk -- interfacing with scsi sub-system to claim and export real
+scsi devices ie. disks, hardware raid volumes, tape library as SRP's luns
+
+2. scst_vdisk -- fileio and blockio modes. This allows you to turn software raid
+volumes, LVM volumes, IDE disks, and normal files into SRP's luns (required to
+work with latest scst's svn tree)
+
+3. NULLIO mode will allow you to measure the performance without sending IOs
+to *real* devices
+
+
+Prerequisites
+-------------
+1. First it is required to download and install SCST driver. Please
+refer to SCST's Internet page http://scst.sourceforge.net for download
+and installation instructions. You can also find SCST's design document,
+svn repository development tree and utility tools for basic storage
+management
+
+a. git clone git://git.openfabrics.org/~vu/srpt_inc ~/srpt_inc
+b. Checking out scst's svn development tree revision 245
+svn co https://scst.svn.sourceforge.net/svnroot/scst/trunk/scst -r 245
+c. cd scst
+d. patch -p0 < ~/srpt_inc/scst_r245.patch
+e. make and make install
+
+2. Download/install IB low-level driver from OFED package or linux kernel tree
+a. SRP target is part of OFED-1.3
+or
+b. Download and install OFED-1.x (x > 1) with kernel_ib development package rpm
+or
+c. Built and installed Infiniband driver in Linux "vanilla" kernel tree
+ from kernel.org
+
+3. Download SRP target driver from openfabrics.org (needed for 2b, 2c)
+git clone git://git.openfabrics.org/~vu/srpt.git ~/srpt
+
+
+Installation
+------------
+A. For OFED-1.3
+----------------
+SRP target is part of ofed-1.3. Go through normal installation with srpt enable
+
+
+B. Instruction to included in and built with OFED-1.2.5 development tree
+-----------------------------------------------------------------------
+1. cd /usr/src/ofa_kernel-1.2.5 or /usr/src/ofa_kernel-1.2.c or
+~/ofa_1_2_c_kernel-200708xx-yyyy
+2. patch -p1 < ~/srpt_inc/add_srpt_01.patch
+3. patch -p1 < ~/srpt_inc/add_srpt_03.patch
+4. patch -p1 < ~/srpt_inc/add_srpt_04.patch
+5. cp -r ~/srpt drivers/infiniband/ulp/srpt
+6. ./configure --with-core-mod --with-mthca-mod --without-mthca_debug-mod
+ --with-srp-target-mod <and any other modules/drivers as needed>
+7. make and make install
+
+C. Instruction to be included in and built with OFED-1.2 development tree
+--------------------------------------------------------------------------
+1. cd /usr/src/ofa_kernel-1.2 or /usr/src/ofa_kernel-1.2 or
+~/ofa_1_2_kernel-200708xx-yyyy
+2. patch -p1 < ~/srpt_inc/add_srpt_01.patch
+3. patch -p1 < ~/srpt_inc/add_srpt_03.patch
+4. patch -p1 < ~/srpt_inc/add_srpt_04.patch
+5. cp -r ~/srpt drivers/infiniband/ulp/srpt
+6. patch -p1 < ~/srpt_inc/add_srpt_ofed_1_2.patch
+7. ./configure --with-core-mod --with-mthca-mod --without-mthca_debug-mod
+ --with-srp-target-mod <and any other modules/drivers as needed>
+8. make and make install
+
+D. Instruction to be included in and built with "vanilla" kernel development tree
+----------------------------------------------------------------------------------
+1. cd /usr/src/linux-2.6.1x
+2. patch -p1 < ~/srpt_inc/add_srpt_03.patch
+3. cp -r ~/srpt drivers/infiniband/ulp/srpt
+4. configure and build SRP target module driver as normal
+
+
+How-to run
+-----------
+A. On srp target machine
+1. Please refer to SCST's README for loading scst driver and its
+dev_handlers drivers (scst_disk, scst_vdisk block or file IO mode, nullio, ...)
+
+Example 1: working with real back-end scsi disks
+a. modprobe scst
+b. modprobe scst_disk
+c. cat /proc/scsi_tgt/scsi_tgt
+
+ibstor00:~ # cat /proc/scsi_tgt/scsi_tgt
+Device (host:ch:id:lun or name) Device handler
+0:0:0:0 dev_disk
+4:0:0:0 dev_disk
+5:0:0:0 dev_disk
+6:0:0:0 dev_disk
+7:0:0:0 dev_disk
+
+Now you want to exclude the first scsi disk and expose the last 4 scsi disks as
+IB/SRP luns for I/O
+echo "add 4:0:0:0 0" >/proc/scsi_tgt/groups/Default/devices
+echo "add 5:0:0:0 1" >/proc/scsi_tgt/groups/Default/devices
+echo "add 6:0:0:0 2" >/proc/scsi_tgt/groups/Default/devices
+echo "add 7:0:0:0 3" >/proc/scsi_tgt/groups/Default/devices
+
+Example 2: working with VDISK FILEIO mode (using md0 device and file 10G-file)
+a. modprobe scst
+b. modprobe scst_vdisk
+c. echo "open vdisk0 /dev/md0" > /proc/scsi_tgt/vdisk/vdisk
+d. echo "open vdisk1 /10G-file" > /proc/scsi_tgt/vdisk/vdisk
+e. echo "add vdisk0 0" >/proc/scsi_tgt/groups/Default/devices
+f. echo "add vdisk1 1" >/proc/scsi_tgt/groups/Default/devices
+
+Example 3: working with VDISK BLOCKIO mode (using md0 device, sda, and cciss/c1d0)
+a. modprobe scst
+b. modprobe scst_vdisk
+c. echo "open vdisk0 /dev/md0 BLOCKIO" > /proc/scsi_tgt/vdisk/vdisk
+d. echo "open vdisk1 /dev/sda BLOCKIO" > /proc/scsi_tgt/vdisk/vdisk
+e. echo "open vdisk2 /dev/cciss/c1d0 BLOCKIO" > /proc/scsi_tgt/vdisk/vdisk
+f. echo "add vdisk0 0" >/proc/scsi_tgt/groups/Default/devices
+g. echo "add vdisk1 1" >/proc/scsi_tgt/groups/Default/devices
+h. echo "add vdisk2 2" >/proc/scsi_tgt/groups/Default/devices
+
+2. modprobe ib_srpt
+
+
+B. On initiator machines you can manualy do the following steps:
+1. modprobe ib_srp
+2. ipsrpdm -c (to discover new SRP target)
+3. echo <new target info> > /sys/class/infiniband_srp/srp-mthca0-1/add_target
+4. fdisk -l (will show new discovered scsi disks)
+
+Example:
+Assume that you use port 1 of first HCA in the system ie. mthca0
+
+[root@lab104 ~]# ibsrpdm -c -d /dev/infiniband/umad0
+id_ext=0002c90200226cf4,ioc_guid=0002c90200226cf4,
+dgid=fe800000000000000002c90200226cf5,pkey=ffff,service_id=0002c90200226cf4
+[root@lab104 ~]# echo id_ext=0002c90200226cf4,ioc_guid=0002c90200226cf4,
+dgid=fe800000000000000002c90200226cf5,pkey=ffff,service_id=0002c90200226cf4 >
+/sys/class/infiniband_srp/srp-mthca0-1/add_target
+
+OR
+
++ You can edit /etc/infiniband/openib.conf to load srp driver and srp HA daemon
+automatically ie. set SRP_LOAD=yes, and SRPHA_ENABLE=yes
++ To set up and use high availability feature you need dm-multipath driver
+and multipath tool
++ Please refer to OFED-1.x SRP's user manual for more in-details instructions
+on how-to enable/use HA feature
+
+
+TO DO
+------
++ Stress test and stabilize the code
++ Performance tuning
Property changes on: trunk/srpt/patches
___________________________________________________________________
Name: svn:ignore
+ *.o
*.mod.c
*.ko
*.ko.cmd
*.o.cmd
.tmp_versions
Modules.symvers
Module.symvers
Added: trunk/srpt/patches/scst_increase_max_tgt_cmds.patch
===================================================================
--- trunk/srpt/patches/scst_increase_max_tgt_cmds.patch (rev 0)
+++ trunk/srpt/patches/scst_increase_max_tgt_cmds.patch 2008-03-17 11:29:50 UTC (rev 304)
@@ -0,0 +1,13 @@
+Index: scst/src/scst_priv.h
+===================================================================
+--- scst/src/scst_priv.h (revision 303)
++++ scst/src/scst_priv.h (working copy)
+@@ -110,7 +110,7 @@ extern unsigned long scst_trace_flag;
+ ** Maximum count of uncompleted commands that an initiator could
+ ** queue on any device. Then it will start getting TASK QUEUE FULL status.
+ **/
+-#define SCST_MAX_TGT_DEV_COMMANDS 48
++#define SCST_MAX_TGT_DEV_COMMANDS 64
+
+ /**
+ ** Maximum count of uncompleted commands that could be queued on any device.
Property changes on: trunk/srpt/src
___________________________________________________________________
Name: svn:ignore
+ *.o
*.mod.c
*.ko
*.ko.cmd
*.o.cmd
.tmp_versions
Modules.symvers
Module.symvers
Added: trunk/srpt/src/Kconfig
===================================================================
--- trunk/srpt/src/Kconfig (rev 0)
+++ trunk/srpt/src/Kconfig 2008-03-17 11:29:50 UTC (rev 304)
@@ -0,0 +1,12 @@
+config INFINIBAND_SRPT
+ tristate "InfiniBand SCSI RDMA Protocol Target Mode"
+ depends on INFINIBAND
+ ---help---
+ Support for the SCSI RDMA Protocol Target mode over InfiniBand.
+ This allows you to turn a standard Linux box to native Infiniband
+ storage using SRP protocol.
+
+ You will also need the SCST middle level drivers from
+ http://scst.sf.net/
+ The SRP protocol is defined by the INCITS T10 technical
+ committee. See <http://www.t10.org/>.
Added: trunk/srpt/src/Makefile
===================================================================
--- trunk/srpt/src/Makefile (rev 0)
+++ trunk/srpt/src/Makefile 2008-03-17 11:29:50 UTC (rev 304)
@@ -0,0 +1,7 @@
+SCST_INC_DIR := $(SUBDIRS)/../../scst/include
+
+EXTRA_CFLAGS += -I$(SCST_INC_DIR)
+EXTRA_CFLAGS += -DDEBUG -g
+#EXTRA_CFLAGS += -Wextra -Wno-unused-parameter
+
+obj-m += ib_srpt.o
Added: trunk/srpt/src/Makefile.in_kernel
===================================================================
--- trunk/srpt/src/Makefile.in_kernel (rev 0)
+++ trunk/srpt/src/Makefile.in_kernel 2008-03-17 11:29:50 UTC (rev 304)
@@ -0,0 +1,4 @@
+EXTRA_CFLAGS += -Idrivers/infiniband/include
+EXTRA_CFLAGS += -I/usr/local/include/scst
+
+obj-$(CONFIG_INFINIBAND_SRPT) += ib_srpt.o
Added: trunk/srpt/src/ib_dm_mad.h
===================================================================
--- trunk/srpt/src/ib_dm_mad.h (rev 0)
+++ trunk/srpt/src/ib_dm_mad.h 2008-03-17 11:29:50 UTC (rev 304)
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2006 Mellanox Technology Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef IB_DM_MAD_H
+#define IB_DM_MAD_H
+
+#include <linux/types.h>
+
+#include <rdma/ib_mad.h>
+
+enum {
+ DM_MAD_STATUS_UNSUP_METHOD = 0x0008,
+ DM_MAD_STATUS_UNSUP_METHOD_ATTR = 0x000c,
+ DM_MAD_STATUS_INVALID_FIELD = 0x001c,
+ DM_MAD_STATUS_NO_IOC = 0x0100,
+
+ DM_ATTR_CLASS_PORT_INFO = 0x01,
+ DM_ATTR_IOU_INFO = 0x10,
+ DM_ATTR_IOC_PROFILE = 0x11,
+ DM_ATTR_SVC_ENTRIES = 0x12
+};
+
+struct ib_dm_hdr {
+ u8 reserved[28];
+};
+
+struct ib_dm_mad {
+ struct ib_mad_hdr mad_hdr;
+ struct ib_rmpp_hdr rmpp_hdr;
+ struct ib_dm_hdr dm_hdr;
+ u8 data[IB_MGMT_DEVICE_DATA];
+};
+
+struct ib_dm_iou_info {
+ __be16 change_id;
+ u8 max_controllers;
+ u8 op_rom;
+ u8 controller_list[128];
+};
+
+struct ib_dm_ioc_profile {
+ __be64 guid;
+ __be32 vendor_id;
+ __be32 device_id;
+ __be16 device_version;
+ __be16 reserved1;
+ __be32 subsys_vendor_id;
+ __be32 subsys_device_id;
+ __be16 io_class;
+ __be16 io_subclass;
+ __be16 protocol;
+ __be16 protocol_version;
+ __be16 service_conn;
+ __be16 initiators_supported;
+ __be16 send_queue_depth;
+ u8 reserved2;
+ u8 rdma_read_depth;
+ __be32 send_size;
+ __be32 rdma_size;
+ u8 op_cap_mask;
+ u8 svc_cap_mask;
+ u8 num_svc_entries;
+ u8 reserved3[9];
+ u8 id_string[64];
+};
+
+struct ib_dm_svc_entry {
+ u8 name[40];
+ u64 id;
+};
+
+struct ib_dm_svc_entries {
+ struct ib_dm_svc_entry service_entries[4];
+};
+
+#endif
Added: trunk/srpt/src/ib_srpt.c
===================================================================
--- trunk/srpt/src/ib_srpt.c (rev 0)
+++ trunk/srpt/src/ib_srpt.c 2008-03-17 11:29:50 UTC (rev 304)
@@ -0,0 +1,2317 @@
+/*
+ * Copyright (c) 2006 Mellanox Technology Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/err.h>
+#include <linux/string.h>
+#include <linux/kthread.h>
+
+#include <asm/atomic.h>
+
+#include "ib_srpt.h"
+
+#define DRV_NAME "ib_srpt"
+#define PFX DRV_NAME ": "
+#define DRV_VERSION "0.1"
+#define DRV_RELDATE "January 10, 2007"
+
+#define MELLANOX_SRPT_ID_STRING "Mellanox OFED SRP target"
+
+MODULE_AUTHOR("Vu Pham");
+MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
+ "v" DRV_VERSION " (" DRV_RELDATE ")");
+MODULE_LICENSE("Dual BSD/GPL");
+
+struct srpt_thread {
+ spinlock_t thread_lock;
+ struct list_head thread_ioctx_list;
+ struct task_struct *thread;
+};
+
+static u64 mellanox_ioc_guid = 0;
+static struct list_head srpt_devices;
+static int thread = 1;
+static struct srpt_thread srpt_thread;
+DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
+
+module_param(thread, int, 0444);
+MODULE_PARM_DESC(thread,
+ "Executing ioctx in thread context. Default thread = 1");
+
+static void srpt_add_one(struct ib_device *device);
+static void srpt_remove_one(struct ib_device *device);
+static int srpt_disconnect_channel(struct srpt_rdma_ch *ch, int dreq);
+
+static struct ib_client srpt_client = {
+ .name = DRV_NAME,
+ .add = srpt_add_one,
+ .remove = srpt_remove_one
+};
+
+static void srpt_event_handler(struct ib_event_handler *handler,
+ struct ib_event *event)
+{
+ struct srpt_device *sdev =
+ ib_get_client_data(event->device, &srpt_client);
+ struct srpt_port *sport;
+
+ if (!sdev || sdev->device != event->device)
+ return;
+
+ printk(KERN_WARNING PFX "ASYNC event= %d on device= %s\n",
+ event->event, sdev->device->name);
+
+ switch (event->event) {
+ case IB_EVENT_PORT_ERR:
+ if (event->element.port_num <= sdev->device->phys_port_cnt) {
+ sport = &sdev->port[event->element.port_num - 1];
+ sport->lid = 0;
+ sport->sm_lid = 0;
+ }
+ break;
+ case IB_EVENT_PORT_ACTIVE:
+ case IB_EVENT_LID_CHANGE:
+ case IB_EVENT_PKEY_CHANGE:
+ case IB_EVENT_SM_CHANGE:
+ case IB_EVENT_CLIENT_REREGISTER:
+ if (event->element.port_num <= sdev->device->phys_port_cnt) {
+ sport = &sdev->port[event->element.port_num - 1];
+ if (!sport->lid && !sport->sm_lid)
+ schedule_work(&sport->work);
+ }
+ break;
+ default:
+ break;
+ }
+
+}
+
+static void srpt_srq_event(struct ib_event *event, void *ctx)
+{
+ printk(KERN_WARNING PFX "SRQ event %d\n", event->event);
+}
+
+static void srpt_qp_event(struct ib_event *event, void *ctx)
+{
+ struct srpt_rdma_ch *ch = ctx;
+
+ printk(KERN_WARNING PFX "QP event %d on cm_id= %p sess_name= %s state= %d\n",
+ event->event, ch->cm_id, ch->sess_name, ch->state);
+
+ switch (event->event) {
+ case IB_EVENT_COMM_EST:
+ ib_cm_notify(ch->cm_id, event->event);
+ break;
+ case IB_EVENT_QP_LAST_WQE_REACHED:
+ if (ch->state == RDMA_CHANNEL_LIVE) {
+ printk(KERN_WARNING PFX "Schedule CM_DISCONNECT_WORK\n");
+ srpt_disconnect_channel(ch, 1);
+ }
+ break;
+ default:
+ break;
+ }
+}
+
+static void srpt_set_ioc(u8 * c_list, u32 slot, u8 value)
+{
+ u16 id;
+ u8 tmp;
+
+ id = (slot - 1) / 2;
+ if (slot & 0x1) {
+ tmp = c_list[id] & 0xf;
+ c_list[id] = (value << 4) | tmp;
+ } else {
+ tmp = c_list[id] & 0xf0;
+ c_list[id] = (value & 0xf) | tmp;
+ }
+}
+
+static void srpt_get_class_port_info(struct ib_dm_mad *mad)
+{
+ struct ib_class_port_info *cif;
+
+ cif = (struct ib_class_port_info *)mad->data;
+ memset(cif, 0, sizeof *cif);
+ cif->base_version = 1;
+ cif->class_version = 1;
+ cif->resp_time_value = 20;
+
+ mad->mad_hdr.status = 0;
+}
+
+static void srpt_get_iou(struct ib_dm_mad *mad)
+{
+ struct ib_dm_iou_info *ioui;
+ u8 slot;
+ int i;
+
+ ioui = (struct ib_dm_iou_info *)mad->data;
+ ioui->change_id = 1;
+ ioui->max_controllers = 16;
+
+ /* set present for slot 1 and empty for the rest */
+ srpt_set_ioc(ioui->controller_list, 1, 1);
+ for (i = 1, slot = 2; i < 16; i++, slot++)
+ srpt_set_ioc(ioui->controller_list, slot, 0);
+
+ mad->mad_hdr.status = 0;
+}
+
+static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
+ struct ib_dm_mad *mad)
+{
+ struct ib_dm_ioc_profile *iocp;
+
+ iocp = (struct ib_dm_ioc_profile *)mad->data;
+
+ if (!slot || slot > 16) {
+ mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
+ return;
+ }
+
+ if (slot > 2) {
+ mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
+ return;
+ }
+
+ memset(iocp, 0, sizeof *iocp);
+ strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
+ iocp->guid = cpu_to_be64(mellanox_ioc_guid);
+ iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
+ iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
+ iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
+ iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
+ iocp->subsys_device_id = 0x0;
+ iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
+ iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
+ iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
+ iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
+ iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
+ iocp->rdma_read_depth = 4;
+ iocp->send_size = cpu_to_be32(MAX_MESSAGE_SIZE);
+ iocp->rdma_size = cpu_to_be32(MAX_RDMA_SIZE);
+ iocp->num_svc_entries = 1;
+ iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
+ SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
+
+ mad->mad_hdr.status = 0;
+}
+
+static void srpt_get_svc_entries(u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
+{
+ struct ib_dm_svc_entries *svc_entries;
+
+ if (!slot || slot > 16) {
+ mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
+ return;
+ }
+
+ if (slot > 2 || lo > hi || hi > 1) {
+ mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
+ return;
+ }
+
+ svc_entries = (struct ib_dm_svc_entries *)mad->data;
+ memset(svc_entries, 0, sizeof *svc_entries);
+ svc_entries->service_entries[0].id = cpu_to_be64(mellanox_ioc_guid);
+ sprintf(svc_entries->service_entries[0].name, "%s%016llx",
+ SRP_SERVICE_NAME_PREFIX, (unsigned long long)mellanox_ioc_guid);
+
+ mad->mad_hdr.status = 0;
+}
+
+static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
+ struct ib_dm_mad *rsp_mad)
+{
+ u16 attr_id;
+ u32 slot;
+ u8 hi, lo;
+
+ attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
+ switch (attr_id) {
+ case DM_ATTR_CLASS_PORT_INFO:
+ srpt_get_class_port_info(rsp_mad);
+ break;
+ case DM_ATTR_IOU_INFO:
+ srpt_get_iou(rsp_mad);
+ break;
+ case DM_ATTR_IOC_PROFILE:
+ slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
+ srpt_get_ioc(sp->sdev, slot, rsp_mad);
+ break;
+ case DM_ATTR_SVC_ENTRIES:
+ slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
+ hi = (u8) ((slot >> 8) & 0xff);
+ lo = (u8) (slot & 0xff);
+ slot = (u16) ((slot >> 16) & 0xffff);
+ srpt_get_svc_entries(slot, hi, lo, rsp_mad);
+ break;
+ default:
+ rsp_mad->mad_hdr.status =
+ cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
+ break;
+ }
+}
+
+static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
+ struct ib_mad_send_wc *mad_wc)
+{
+ ib_destroy_ah(mad_wc->send_buf->ah);
+ ib_free_send_mad(mad_wc->send_buf);
+}
+
+static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
+ struct ib_mad_recv_wc *mad_wc)
+{
+ struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
+ struct ib_ah *ah;
+ struct ib_mad_send_buf *rsp;
+ struct ib_dm_mad *dm_mad;
+
+ if (!mad_wc || !mad_wc->recv_buf.mad)
+ return;
+
+ ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
+ mad_wc->recv_buf.grh, mad_agent->port_num);
+ if (IS_ERR(ah))
+ goto err;
+
+ rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
+ mad_wc->wc->pkey_index, 0,
+ IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
+ GFP_KERNEL);
+ if (IS_ERR(rsp))
+ goto err_rsp;
+
+ rsp->ah = ah;
+
+ dm_mad = rsp->mad;
+ memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
+ dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
+ dm_mad->mad_hdr.status = 0;
+
+ switch (mad_wc->recv_buf.mad->mad_hdr.method) {
+ case IB_MGMT_METHOD_GET:
+ srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
+ break;
+ case IB_MGMT_METHOD_SET:
+ dm_mad->mad_hdr.status =
+ cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
+ break;
+ default:
+ dm_mad->mad_hdr.status =
+ cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
+ break;
+ }
+
+ if (!ib_post_send_mad(rsp, NULL)) {
+ ib_free_recv_mad(mad_wc);
+ /* will destroy_ah & free_send_mad in send completion */
+ return;
+ }
+
+ ib_free_send_mad(rsp);
+
+ err_rsp:
+ ib_destroy_ah(ah);
+ err:
+ ib_free_recv_mad(mad_wc);
+}
+
+static int srpt_refresh_port(struct srpt_port *sport)
+{
+ struct ib_mad_reg_req reg_req;
+ struct ib_port_modify port_modify;
+ struct ib_port_attr port_attr;
+ int ret;
+
+ memset(&port_modify, 0, sizeof port_modify);
+ port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
+ port_modify.clr_port_cap_mask = 0;
+
+ ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
+ if (ret)
+ goto err_mod_port;
+
+ ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
+ if (ret)
+ goto err_query_port;
+
+ sport->sm_lid = port_attr.sm_lid;
+ sport->lid = port_attr.lid;
+
+ ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
+ if (ret)
+ goto err_query_port;
+
+ if (!sport->mad_agent) {
+ memset(®_req, 0, sizeof reg_req);
+ reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
+ reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
+ set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
+ set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
+
+ sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
+ sport->port,
+ IB_QPT_GSI,
+ ®_req, 0,
+ srpt_mad_send_handler,
+ srpt_mad_recv_handler,
+ sport);
+ if (IS_ERR(sport->mad_agent))
+ goto err_query_port;
+ }
+
+ return 0;
+
+ err_query_port:
+
+ port_modify.set_port_cap_mask = 0;
+ port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
+ ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
+
+ err_mod_port:
+
+ return ret;
+}
+
+static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
+{
+ struct srpt_ioctx *ioctx;
+
+ ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
+ if (!ioctx)
+ goto out;
+
+ ioctx->buf = kzalloc(MAX_MESSAGE_SIZE, GFP_KERNEL);
+ if (!ioctx->buf)
+ goto out_free_ioctx;
+
+ ioctx->dma = dma_map_single(sdev->device->dma_device, ioctx->buf,
+ MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(ioctx->dma))
+ goto out_free_buf;
+
+ return ioctx;
+
+ out_free_buf:
+ kfree(ioctx->buf);
+ out_free_ioctx:
+ kfree(ioctx);
+ out:
+ return NULL;
+}
+
+static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
+{
+ if (!ioctx)
+ return;
+
+ dma_unmap_single(sdev->device->dma_device, ioctx->dma,
+ MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
+ kfree(ioctx->buf);
+ kfree(ioctx);
+}
+
+static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
+{
+ int i;
+
+ for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
+ sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
+
+ if (!sdev->ioctx_ring[i])
+ goto err;
+
+ sdev->ioctx_ring[i]->index = i;
+ }
+
+ return 0;
+
+ err:
+ while (--i > 0) {
+ srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
+ sdev->ioctx_ring[i] = NULL;
+ }
+ return -ENOMEM;
+}
+
+static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
+{
+ struct ib_sge list;
+ struct ib_recv_wr wr, *bad_wr;
+
+ wr.wr_id = ioctx->index | SRPT_OP_RECV;
+
+ list.addr = ioctx->dma;
+ list.length = MAX_MESSAGE_SIZE;
+ list.lkey = sdev->mr->lkey;
+
+ wr.next = NULL;
+ wr.sg_list = &list;
+ wr.num_sge = 1;
+
+ return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
+}
+
+static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
+ int len)
+{
+ struct ib_sge list;
+ struct ib_send_wr wr, *bad_wr;
+ struct srpt_device *sdev = ch->sport->sdev;
+
+ dma_sync_single_for_device(sdev->device->dma_device, ioctx->dma,
+ MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
+
+ list.addr = ioctx->dma;
+ list.length = len;
+ list.lkey = sdev->mr->lkey;
+
+ wr.next = NULL;
+ wr.wr_id = ioctx->index;
+ wr.sg_list = &list;
+ wr.num_sge = 1;
+ wr.opcode = IB_WR_SEND;
+ wr.send_flags = IB_SEND_SIGNALED;
+
+ return ib_post_send(ch->qp, &wr, &bad_wr);
+}
+
+static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
+ int *ind)
+{
+ struct srp_indirect_buf *idb;
+ struct srp_direct_buf *db;
+
+ *ind = 0;
+ if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
+ ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
+ ioctx->n_rbuf = 1;
+ ioctx->rbufs = &ioctx->single_rbuf;
+
+ db = (void *)srp_cmd->add_data;
+ memcpy(ioctx->rbufs, db, sizeof *db);
+ ioctx->data_len = be32_to_cpu(db->len);
+ } else {
+ idb = (void *)srp_cmd->add_data;
+
+ ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
+
+ if (ioctx->n_rbuf >
+ (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
+ *ind = 1;
+ ioctx->n_rbuf = 0;
+ goto out;
+ }
+
+ if (ioctx->n_rbuf == 1)
+ ioctx->rbufs = &ioctx->single_rbuf;
+ else
+ ioctx->rbufs = kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
+ if (!ioctx->rbufs) {
+ ioctx->n_rbuf = 0;
+ return -ENOMEM;
+ }
+
+ db = idb->desc_list;
+ memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
+ ioctx->data_len = be32_to_cpu(idb->len);
+ }
+ out:
+ return 0;
+}
+
+static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
+{
+ struct ib_qp_attr *attr;
+ int ret;
+
+ attr = kzalloc(sizeof *attr, GFP_KERNEL);
+ if (!attr)
+ return -ENOMEM;
+
+ attr->qp_state = IB_QPS_INIT;
+ attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
+ IB_ACCESS_REMOTE_WRITE;
+ attr->port_num = ch->sport->port;
+ attr->pkey_index = 0;
+
+ ret = ib_modify_qp(qp, attr,
+ IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
+ IB_QP_PKEY_INDEX);
+
+ kfree(attr);
+ return ret;
+}
+
+static int srpt_ch_qp_rtr_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp,
+ enum ib_qp_state qp_state)
+{
+ struct ib_qp_attr *qp_attr;
+ int attr_mask;
+ int ret;
+
+ qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
+ if (!qp_attr)
+ return -ENOMEM;
+
+ qp_attr->qp_state = qp_state;
+ ret = ib_cm_init_qp_attr(ch->cm_id, qp_attr, &attr_mask);
+ if (ret)
+ goto out;
+
+ if (qp_state == IB_QPS_RTR)
+ qp_attr->max_dest_rd_atomic = 4;
+ else
+ qp_attr->max_rd_atomic = 4;
+
+ ret = ib_modify_qp(qp, qp_attr, attr_mask);
+
+ out:
+ kfree(qp_attr);
+ return ret;
+}
+
+static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
+{
+ int i;
+
+ if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
+ struct rdma_iu *riu = ioctx->rdma_ius;
+
+ for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu) {
+ if (riu->sge)
+ kfree(riu->sge);
+ }
+ kfree(ioctx->rdma_ius);
+ }
+
+ if (ioctx->n_rbuf > 1 && ioctx->rbufs)
+ kfree(ioctx->rbufs);
+
+ if (srpt_post_recv(ch->sport->sdev, ioctx))
+ printk(KERN_ERR PFX "SRQ post_recv failed - this is serious\n");
+ /* we should queue it back to free_ioctx queue */
+ else
+ atomic_inc(&ch->req_lim_delta);
+}
+
+static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
+{
+ struct srpt_ioctx *ioctx;
+ struct srpt_device *sdev = ch->sport->sdev;
+ scst_data_direction dir = SCST_DATA_NONE;
+
+ if (wc->wr_id & SRPT_OP_RECV) {
+ ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
+ printk(KERN_ERR PFX "This is serious - SRQ is in bad state\n");
+ } else {
+ ioctx = sdev->ioctx_ring[wc->wr_id];
+
+ if (ioctx->scmnd) {
+ struct scst_cmd *scmnd = ioctx->scmnd;
+
+ dir = scst_cmd_get_data_direction(scmnd);
+
+ if (dir == SCST_DATA_NONE)
+ scst_tgt_cmd_done(scmnd);
+ else {
+ dma_unmap_sg(sdev->device->dma_device,
+ scst_cmd_get_sg(scmnd),
+ scst_cmd_get_sg_cnt(scmnd),
+ scst_to_tgt_dma_dir(dir));
+
+ if (scmnd->data_buf_tgt_alloc &&
+ scmnd->data_buf_alloced) {
+ kfree(scmnd->sg);
+ scmnd->sg = NULL;
+ scmnd->sg_cnt = 0;
+ }
+
+ if (scmnd->state == SCST_CMD_STATE_RDY_TO_XFER)
+ scst_rx_data(scmnd,
+ SCST_RX_STATUS_ERROR,
+ SCST_CONTEXT_THREAD);
+ else if (scmnd->state == SCST_CMD_STATE_XMIT_WAIT)
+ scst_tgt_cmd_done(scmnd);
+ }
+ } else
+ srpt_reset_ioctx(ch, ioctx);
+ }
+
+}
+
+static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
+ struct srpt_ioctx *ioctx)
+{
+ if (ioctx->scmnd) {
+ scst_data_direction dir = scst_cmd_get_data_direction(ioctx->scmnd);
+
+ if (dir != SCST_DATA_NONE)
+ dma_unmap_sg(ch->sport->sdev->device->dma_device,
+ scst_cmd_get_sg(ioctx->scmnd),
+ scst_cmd_get_sg_cnt(ioctx->scmnd),
+ scst_to_tgt_dma_dir(dir));
+
+ if (ioctx->scmnd->data_buf_tgt_alloc &&
+ ioctx->scmnd->data_buf_alloced) {
+ kfree(ioctx->scmnd->sg);
+ ioctx->scmnd->sg = NULL;
+ ioctx->scmnd->sg_cnt = 0;
+ }
+
+ scst_tgt_cmd_done(ioctx->scmnd);
+ } else
+ srpt_reset_ioctx(ch, ioctx);
+}
+
+static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
+ struct srpt_ioctx *ioctx)
+{
+ if (!ioctx->scmnd) {
+ srpt_reset_ioctx(ch, ioctx);
+ return;
+ }
+
+ if (scst_cmd_get_data_direction(ioctx->scmnd) == SCST_DATA_WRITE)
+ scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
+ SCST_CONTEXT_THREAD);
+}
+
+static void srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
+ struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
+ u64 tag)
+{
+ struct srp_rsp *srp_rsp;
+ struct sense_data *sense;
+ int limit_delta;
+
+ srp_rsp = ioctx->buf;
+ memset(srp_rsp, 0, sizeof *srp_rsp);
+
+ limit_delta = atomic_read(&ch->req_lim_delta);
+ atomic_sub(limit_delta, &ch->req_lim_delta);
+
+ srp_rsp->opcode = SRP_RSP;
+ srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
+ srp_rsp->tag = tag;
+
+ if (s_key != NO_SENSE) {
+ srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
+ srp_rsp->status = SAM_STAT_CHECK_CONDITION;
+ srp_rsp->sense_data_len =
+ cpu_to_be32(sizeof *sense + (sizeof *sense % 4));
+
+ sense = (struct sense_data *)(srp_rsp + 1);
+ sense->err_code = 0x70;
+ sense->key = s_key;
+ sense->asc_ascq = s_code;
+ }
+}
+
+static void srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
+ struct srpt_ioctx *ioctx, u8 rsp_code,
+ u64 tag)
+{
+ struct srp_rsp *srp_rsp;
+ int limit_delta;
+
+ dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
+ MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
+
+ srp_rsp = ioctx->buf;
+ memset(srp_rsp, 0, sizeof *srp_rsp);
+
+ limit_delta = atomic_read(&ch->req_lim_delta);
+ atomic_sub(limit_delta, &ch->req_lim_delta);
+
+ srp_rsp->opcode = SRP_RSP;
+ srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
+ srp_rsp->tag = tag;
+
+ if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
+ srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
+ srp_rsp->resp_data_len = cpu_to_be32(4);
+ srp_rsp->data[3] = rsp_code;
+ }
+}
+
+static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
+ struct srpt_ioctx *ioctx)
+{
+ struct scst_cmd *scmnd = NULL;
+ struct srp_cmd *srp_cmd = NULL;
+ struct srp_tsk_mgmt *srp_tsk = NULL;
+ struct srpt_mgmt_ioctx *mgmt_ioctx;
+ scst_data_direction dir = SCST_DATA_NONE;
+ int indirect_desc = 0;
+ u8 op;
+ int ret;
+
+ if (ch->state != RDMA_CHANNEL_LIVE) {
+ if (ch->state == RDMA_CHANNEL_CONNECTING) {
+ spin_lock_irq(&ch->spinlock);
+ list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
+ spin_unlock_irq(&ch->spinlock);
+ } else
+ srpt_reset_ioctx(ch, ioctx);
+
+ return;
+ }
+
+ dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
+ MAX_MESSAGE_SIZE, DMA_FROM_DEVICE);
+
+ ioctx->data_len = 0;
+ ioctx->n_rbuf = 0;
+ ioctx->rbufs = NULL;
+ ioctx->n_rdma = 0;
+ ioctx->n_rdma_ius = 0;
+ ioctx->rdma_ius = NULL;
+ ioctx->scmnd = NULL;
+
+ op = *(u8 *) ioctx->buf;
+ switch (op) {
+ case SRP_CMD:
+ srp_cmd = ioctx->buf;
+
+ if (srp_cmd->buf_fmt) {
+ ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
+ if (ret) {
+ srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
+ NO_ADD_SENSE, srp_cmd->tag);
+ ((struct srp_rsp*)ioctx->buf)->status =
+ SAM_STAT_TASK_SET_FULL;
+ goto send_rsp;
+ }
+
+ if (indirect_desc) {
+ srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
+ NO_ADD_SENSE, srp_cmd->tag);
+ ((struct srp_rsp*)ioctx->buf)->status =
+ SAM_STAT_TASK_SET_FULL;
+ goto send_rsp;
+ }
+
+ if (srp_cmd->buf_fmt & 0xf)
+ dir = SCST_DATA_READ;
+ else if (srp_cmd->buf_fmt >> 4)
+ dir = SCST_DATA_WRITE;
+ else
+ dir = SCST_DATA_NONE;
+ } else
+ dir = SCST_DATA_NONE;
+
+ scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) & srp_cmd->lun,
+ sizeof srp_cmd->lun, srp_cmd->cdb, 16,
+ thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
+ if (!scmnd) {
+ srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
+ NO_ADD_SENSE, srp_cmd->tag);
+ ((struct srp_rsp*)ioctx->buf)->status =
+ SAM_STAT_TASK_SET_FULL;
+ goto send_rsp;
+ }
+
+ ioctx->scmnd = scmnd;
+
+ switch (srp_cmd->task_attr) {
+ case SRP_CMD_HEAD_OF_Q:
+ scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
+ break;
+ case SRP_CMD_ORDERED_Q:
+ scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
+ break;
+ case SRP_CMD_SIMPLE_Q:
+ scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
+ break;
+ case SRP_CMD_ACA:
+ scmnd->queue_type = SCST_CMD_QUEUE_ACA;
+ break;
+ default:
+ scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
+ break;
+ }
+
+ scst_cmd_set_tag(scmnd, srp_cmd->tag);
+ scst_cmd_set_tgt_priv(scmnd, ioctx);
+ scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
+
+ spin_lock_irq(&ch->spinlock);
+ list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
+ ch->active_scmnd_cnt++;
+ scst_cmd_init_done(scmnd, SCST_CONTEXT_THREAD);
+ spin_unlock_irq(&ch->spinlock);
+
+ break;
+
+ case SRP_TSK_MGMT:
+ srp_tsk = ioctx->buf;
+
+ printk(KERN_WARNING PFX
+ "recv_tsk_mgmt= %d for task_tag= %lld"
+ " using tag= %lld cm_id= %p sess= %p\n",
+ srp_tsk->tsk_mgmt_func,
+ (unsigned long long) srp_tsk->task_tag,
+ (unsigned long long) srp_tsk->tag,
+ ch->cm_id, ch->scst_sess);
+
+ mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
+ if (!mgmt_ioctx) {
+ srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
+ srp_tsk->tag);
+ goto send_rsp;
+ }
+
+ mgmt_ioctx->ioctx = ioctx;
+ mgmt_ioctx->ch = ch;
+ mgmt_ioctx->tag = srp_tsk->tag;
+
+ switch (srp_tsk->tsk_mgmt_func) {
+ case SRP_TSK_ABORT_TASK:
+ ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
+ SCST_ABORT_TASK,
+ srp_tsk->task_tag,
+ thread ? SCST_NON_ATOMIC : SCST_ATOMIC,
+ mgmt_ioctx);
+ break;
+ case SRP_TSK_ABORT_TASK_SET:
+ ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
+ SCST_ABORT_TASK_SET,
+ (u8 *) & srp_tsk->lun,
+ sizeof srp_tsk->lun,
+ thread ? SCST_NON_ATOMIC : SCST_ATOMIC,
+ mgmt_ioctx);
+ break;
+ case SRP_TSK_CLEAR_TASK_SET:
+ ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
+ SCST_CLEAR_TASK_SET,
+ (u8 *) & srp_tsk->lun,
+ sizeof srp_tsk->lun,
+ thread ? SCST_NON_ATOMIC : SCST_ATOMIC,
+ mgmt_ioctx);
+ break;
+#if 0
+ case SRP_TSK_LUN_RESET:
+ ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
+ SCST_LUN_RESET,
+ (u8 *) & srp_tsk->lun,
+ sizeof srp_tsk->lun,
+ thread ? SCST_NON_ATOMIC : SCST_ATOMIC,
+ mgmt_ioctx);
+ break;
+#endif
+ case SRP_TSK_CLEAR_ACA:
+ ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
+ SCST_CLEAR_ACA,
+ (u8 *) & srp_tsk->lun,
+ sizeof srp_tsk->lun,
+ thread ? SCST_NON_ATOMIC : SCST_ATOMIC,
+ mgmt_ioctx);
+ break;
+ default:
+ srpt_build_tskmgmt_rsp(ch, ioctx,
+ SRP_TSK_MGMT_FUNC_NOT_SUPP,
+ srp_tsk->tag);
+ goto send_rsp;
+ }
+
+ break;
+ case SRP_I_LOGOUT:
+ case SRP_AER_REQ:
+ default:
+ srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
+ ((struct srp_cmd *)ioctx->buf)->tag);
+
+ goto send_rsp;
+ }
+
+ dma_sync_single_for_device(ch->sport->sdev->device->dma_device,
+ ioctx->dma, MAX_MESSAGE_SIZE,
+ DMA_FROM_DEVICE);
+
+ return;
+
+ send_rsp:
+ if (ch->state != RDMA_CHANNEL_LIVE ||
+ srpt_post_send(ch, ioctx,
+ sizeof(struct srp_rsp) +
+ be32_to_cpu(((struct srp_rsp *)ioctx->buf)->
+ sense_data_len)))
+ srpt_reset_ioctx(ch, ioctx);
+}
+
+static inline int srpt_test_ioctx_list(void)
+{
+ int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
+ unlikely(kthread_should_stop()));
+ return res;
+}
+
+static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&srpt_thread.thread_lock, flags);
+ list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
+ spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
+ wake_up(&ioctx_list_waitQ);
+}
+
+static void srpt_completion(struct ib_cq *cq, void *ctx)
+{
+ struct srpt_rdma_ch *ch = ctx;
+ struct srpt_device *sdev = ch->sport->sdev;
+ struct ib_wc wc;
+ struct srpt_ioctx *ioctx;
+
+ ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
+ while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
+ if (wc.status) {
+ printk(KERN_ERR PFX "failed %s status= %d\n",
+ wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
+ wc.status);
+ srpt_handle_err_comp(ch, &wc);
+ break;
+ }
+
+ if (wc.wr_id & SRPT_OP_RECV) {
+ ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
+ if (thread) {
+ ioctx->ch = ch;
+ ioctx->op = IB_WC_RECV;
+ srpt_schedule_thread(ioctx);
+ } else
+ srpt_handle_new_iu(ch, ioctx);
+ continue;
+ } else
+ ioctx = sdev->ioctx_ring[wc.wr_id];
+
+ if (thread) {
+ ioctx->ch = ch;
+ ioctx->op = wc.opcode;
+ srpt_schedule_thread(ioctx);
+ } else {
+ switch (wc.opcode) {
+ case IB_WC_SEND:
+ srpt_handle_send_comp(ch, ioctx);
+ break;
+ case IB_WC_RDMA_WRITE:
+ case IB_WC_RDMA_READ:
+ srpt_handle_rdma_comp(ch, ioctx);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+}
+
+static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
+{
+ struct ib_qp_init_attr *qp_init;
+ struct srpt_device *sdev = ch->sport->sdev;
+ int cqe;
+ int ret;
+
+ qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
+ if (!qp_init)
+ return -ENOMEM;
+
+ cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
+ ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
+#else
+ ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
+#endif
+ if (IS_ERR(ch->cq)) {
+ ret = PTR_ERR(ch->cq);
+ printk(KERN_ERR PFX "failed to create_cq cqe= %d ret= %d\n",
+ cqe, ret);
+ goto out;
+ }
+
+ ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
+
+ qp_init->qp_context = (void *)ch;
+ qp_init->event_handler = srpt_qp_event;
+ qp_init->send_cq = ch->cq;
+ qp_init->recv_cq = ch->cq;
+ qp_init->srq = sdev->srq;
+ qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
+ qp_init->qp_type = IB_QPT_RC;
+ qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
+ qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
+
+ ch->qp = ib_create_qp(sdev->pd, qp_init);
+ if (IS_ERR(ch->qp)) {
+ ret = PTR_ERR(ch->qp);
+ ib_destroy_cq(ch->cq);
+ printk(KERN_ERR PFX "failed to create_qp ret= %d\n", ret);
+ goto out;
+ }
+
+ printk(KERN_DEBUG PFX "%s[%d] max_cqe= %d max_sge= %d cm_id= %p\n",
+ __FUNCTION__, __LINE__, ch->cq->cqe, qp_init->cap.max_send_sge,
+ ch->cm_id);
+
+ ret = srpt_init_ch_qp(ch, ch->qp);
+ if (ret) {
+ ib_destroy_qp(ch->qp);
+ ib_destroy_cq(ch->cq);
+ goto out;
+ }
+
+ atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
+ out:
+ kfree(qp_init);
+ return ret;
+}
+
+static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id)
+{
+ struct srpt_device *sdev = cm_id->context;
+ struct srpt_rdma_ch *ch, *tmp_ch;
+
+ spin_lock_irq(&sdev->spinlock);
+ list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
+ if (ch->cm_id == cm_id) {
+ spin_unlock_irq(&sdev->spinlock);
+ return ch;
+ }
+ }
+
+ spin_unlock_irq(&sdev->spinlock);
+
+ return NULL;
+}
+
+static int srpt_release_channel(struct srpt_rdma_ch *ch, int destroy_cmid)
+{
+ spin_lock_irq(&ch->sport->sdev->spinlock);
+ list_del(&ch->list);
+ spin_unlock_irq(&ch->sport->sdev->spinlock);
+
+ if (ch->cm_id && destroy_cmid) {
+ printk(KERN_WARNING PFX
+ "%s Destroy cm_id= %p\n", __FUNCTION__, ch->cm_id);
+ ib_destroy_cm_id(ch->cm_id);
+ ch->cm_id = NULL;
+ }
+
+ ib_destroy_qp(ch->qp);
+ ib_destroy_cq(ch->cq);
+
+ if (ch->scst_sess) {
+ struct srpt_ioctx *ioctx, *ioctx_tmp;
+
+ printk(KERN_WARNING PFX
+ "%s: Release sess= %p sess_name= %s active_cmd= %d\n",
+ __FUNCTION__, ch->scst_sess, ch->sess_name,
+ ch->active_scmnd_cnt);
+
+ list_for_each_entry_safe(ioctx, ioctx_tmp,
+ &ch->active_scmnd_list, scmnd_list) {
+ list_del(&ioctx->scmnd_list);
+ ch->active_scmnd_cnt--;
+ }
+
+ scst_unregister_session(ch->scst_sess, 0, NULL);
+ ch->scst_sess = NULL;
+ }
+
+ kfree(ch);
+
+ return (destroy_cmid ? 0 : 1);
+}
+
+static void srpt_register_channel_done(struct scst_session *scst_sess,
+ void *data, int status)
+{
+ struct srpt_rdma_ch *ch = data;
+
+ BUG_ON(!ch);
+
+ if (status) {
+ if (ch->scst_sess) {
+ scst_unregister_session(ch->scst_sess, 0, NULL);
+ ch->scst_sess = NULL;
+ }
+ printk(KERN_ERR PFX
+ "%s[%d] Failed to establish sess= %p status= %d\n",
+ __FUNCTION__, __LINE__, scst_sess, status);
+ }
+
+ complete(&ch->scst_sess_done);
+}
+
+static int srpt_disconnect_channel(struct srpt_rdma_ch *ch, int dreq)
+{
+ spin_lock_irq(&ch->spinlock);
+ ch->state = RDMA_CHANNEL_DISCONNECTING;
+ spin_unlock_irq(&ch->spinlock);
+
+ if (dreq)
+ ib_send_cm_dreq(ch->cm_id, NULL, 0);
+ else
+ ib_send_cm_drep(ch->cm_id, NULL, 0);
+
+ return 0;
+}
+
+static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
+ struct ib_cm_req_event_param *param,
+ void *private_data)
+{
+ struct srpt_device *sdev = cm_id->context;
+ struct srp_login_req *req;
+ struct srp_login_rsp *rsp;
+ struct srp_login_rej *rej;
+ struct ib_cm_rep_param *rep_param;
+ struct srpt_rdma_ch *ch, *tmp_ch;
+ u32 it_iu_len;
+ int ret = 0;
+
+ if (!sdev || !private_data)
+ return -EINVAL;
+
+ rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
+ rej = kzalloc(sizeof *rej, GFP_KERNEL);
+ rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
+
+ if (!rsp || !rej || !rep_param) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ req = (struct srp_login_req *)private_data;
+
+ it_iu_len = be32_to_cpu(req->req_it_iu_len);
+
+ printk(KERN_ERR PFX
+ "Host login i_port_id=0x%llx:0x%llx t_port_id=0x%llx:0x%llx"
+ " it_iu_len=%d\n",
+ (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
+ (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
+ (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
+ (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]), it_iu_len);
+
+ if (it_iu_len > MAX_MESSAGE_SIZE || it_iu_len < 64) {
+ rej->reason =
+ cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
+ ret = -EINVAL;
+ printk(KERN_WARNING PFX
+ "Reject invalid it_iu_len=%d\n", it_iu_len);
+ goto reject;
+ }
+
+ if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
+ rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
+
+ spin_lock_irq(&sdev->spinlock);
+
+ list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
+ if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
+ && !memcmp(ch->t_port_id, req->target_port_id, 16)
+ && param->port == ch->sport->port
+ && param->listen_id == ch->sport->sdev->cm_id
+ && ch->cm_id) {
+ /* found an existing channel */
+ printk(KERN_WARNING PFX
+ "Found existing channel name= %s"
+ " cm_id= %p state= %d\n",
+ ch->sess_name, ch->cm_id, ch->state);
+
+ spin_unlock_irq(&sdev->spinlock);
+
+ rsp->rsp_flags =
+ SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
+
+ if (ch->state == RDMA_CHANNEL_LIVE)
+ srpt_disconnect_channel(ch, 1);
+ else if (ch->state == RDMA_CHANNEL_CONNECTING) {
+ ib_send_cm_rej(ch->cm_id,
+ IB_CM_REJ_NO_RESOURCES,
+ NULL, 0, NULL, 0);
+ srpt_release_channel(ch, 1);
+ }
+
+ spin_lock_irq(&sdev->spinlock);
+ }
+ }
+
+ spin_unlock_irq(&sdev->spinlock);
+
+ } else
+ rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
+
+ if (((u64) (*(u64 *) req->target_port_id) !=
+ cpu_to_be64(mellanox_ioc_guid)) ||
+ ((u64) (*(u64 *) (req->target_port_id + 8)) !=
+ cpu_to_be64(mellanox_ioc_guid))) {
+ rej->reason =
+ cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
+ ret = -ENOMEM;
+ printk(KERN_WARNING PFX "Reject invalid target_port_id\n");
+ goto reject;
+ }
+
+ ch = kzalloc(sizeof *ch, GFP_KERNEL);
+ if (!ch) {
+ rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
+ printk(KERN_WARNING PFX "Reject failed allocate rdma_ch\n");
+ ret = -ENOMEM;
+ goto reject;
+ }
+
+ spin_lock_init(&ch->spinlock);
+ memcpy(ch->i_port_id, req->initiator_port_id, 16)...
[truncated message content] |
