Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 4b5e5f41c8e114bb856347134657eb9e1cccc822

Authored by Bart Van Assche
Committed by Roland Dreier
1 parent 96fc248a4c
Exists in smarc-imx_3.14.28_1.0.0_ga and in 1 other branch imx_3.14.28_1.0.0_ga

IB/srp: Make HCA completion vector configurable 

Several InfiniBand HCAs allow configuring the completion vector per
CQ.  This allows spreading the workload created by IB completion
interrupts over multiple MSI-X vectors and hence over multiple CPU
cores.  In other words, configuring the completion vector properly not
only allows reducing latency on an initiator connected to multiple
SRP targets but also allows improving throughput.

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Acked-by: David Dillow <dillowda@ornl.gov>
Signed-off-by: Roland Dreier <roland@purestorage.com>

Showing 3 changed files with 32 additions and 2 deletions Side-by-side Diff

Documentation/ABI/stable/sysfs-driver-ib_srp
Diff comments   View file @ 4b5e5f4
... ... @@ -54,6 +54,13 @@
54 54 ib_srp. Specifying a value that exceeds cmd_sg_entries is
55 55 only safe with partial memory descriptor list support enabled
56 56 (allow_ext_sg=1).
  57 + * comp_vector, a number in the range 0..n-1 specifying the
  58 + MSI-X completion vector. Some HCA's allocate multiple (n)
  59 + MSI-X vectors per HCA port. If the IRQ affinity masks of
  60 + these interrupts have been configured such that each MSI-X
  61 + interrupt is handled by a different CPU then the comp_vector
  62 + parameter can be used to spread the SRP completion workload
  63 + over multiple CPU's.
57 64  
58 65 What: /sys/class/infiniband_srp/srp-<hca>-<port_number>/ibdev
59 66 Date: January 2, 2006
drivers/infiniband/ulp/srp/ib_srp.c
Diff comments   View file @ 4b5e5f4
... ... @@ -231,14 +231,16 @@
231 231 return -ENOMEM;
232 232  
233 233 recv_cq = ib_create_cq(target->srp_host->srp_dev->dev,
234   - srp_recv_completion, NULL, target, SRP_RQ_SIZE, 0);
  234 + srp_recv_completion, NULL, target, SRP_RQ_SIZE,
  235 + target->comp_vector);
235 236 if (IS_ERR(recv_cq)) {
236 237 ret = PTR_ERR(recv_cq);
237 238 goto err;
238 239 }
239 240  
240 241 send_cq = ib_create_cq(target->srp_host->srp_dev->dev,
241   - srp_send_completion, NULL, target, SRP_SQ_SIZE, 0);
  242 + srp_send_completion, NULL, target, SRP_SQ_SIZE,
  243 + target->comp_vector);
242 244 if (IS_ERR(send_cq)) {
243 245 ret = PTR_ERR(send_cq);
244 246 goto err_recv_cq;
... ... @@ -1898,6 +1900,14 @@
1898 1900 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
1899 1901 }
1900 1902  
  1903 +static ssize_t show_comp_vector(struct device *dev,
  1904 + struct device_attribute *attr, char *buf)
  1905 +{
  1906 + struct srp_target_port *target = host_to_target(class_to_shost(dev));
  1907 +
  1908 + return sprintf(buf, "%d\n", target->comp_vector);
  1909 +}
  1910 +
1901 1911 static ssize_t show_cmd_sg_entries(struct device *dev,
1902 1912 struct device_attribute *attr, char *buf)
1903 1913 {
... ... @@ -1924,6 +1934,7 @@
1924 1934 static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
1925 1935 static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
1926 1936 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
  1937 +static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL);
1927 1938 static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);
1928 1939 static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL);
1929 1940  
... ... @@ -1938,6 +1949,7 @@
1938 1949 &dev_attr_zero_req_lim,
1939 1950 &dev_attr_local_ib_port,
1940 1951 &dev_attr_local_ib_device,
  1952 + &dev_attr_comp_vector,
1941 1953 &dev_attr_cmd_sg_entries,
1942 1954 &dev_attr_allow_ext_sg,
1943 1955 NULL
... ... @@ -2061,6 +2073,7 @@
2061 2073 SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
2062 2074 SRP_OPT_ALLOW_EXT_SG = 1 << 10,
2063 2075 SRP_OPT_SG_TABLESIZE = 1 << 11,
  2076 + SRP_OPT_COMP_VECTOR = 1 << 12,
2064 2077 SRP_OPT_ALL = (SRP_OPT_ID_EXT |
2065 2078 SRP_OPT_IOC_GUID |
2066 2079 SRP_OPT_DGID |
... ... @@ -2081,6 +2094,7 @@
2081 2094 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
2082 2095 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" },
2083 2096 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" },
  2097 + { SRP_OPT_COMP_VECTOR, "comp_vector=%u" },
2084 2098 { SRP_OPT_ERR, NULL }
2085 2099 };
2086 2100  
... ... @@ -2234,6 +2248,14 @@
2234 2248 goto out;
2235 2249 }
2236 2250 target->sg_tablesize = token;
  2251 + break;
  2252 +
  2253 + case SRP_OPT_COMP_VECTOR:
  2254 + if (match_int(args, &token) || token < 0) {
  2255 + pr_warn("bad comp_vector parameter '%s'\n", p);
  2256 + goto out;
  2257 + }
  2258 + target->comp_vector = token;
2237 2259 break;
2238 2260  
2239 2261 default:
drivers/infiniband/ulp/srp/ib_srp.h
Diff comments   View file @ 4b5e5f4
... ... @@ -156,6 +156,7 @@
156 156 char target_name[32];
157 157 unsigned int scsi_id;
158 158 unsigned int sg_tablesize;
  159 + int comp_vector;
159 160  
160 161 struct ib_sa_path_rec path;
161 162 __be16 orig_dgid[8];