Blame view
net/rds/ib_rdma.c
17.4 KB
08b48a1ed RDS/IB: Implement... |
1 |
/* |
b7ff8b103 rds: Extend RDS A... |
2 |
* Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved. |
08b48a1ed RDS/IB: Implement... |
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
* * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #include <linux/kernel.h> |
5a0e3ad6a include cleanup: ... |
34 |
#include <linux/slab.h> |
764f2dd92 rds: rcu-ize rds_... |
35 |
#include <linux/rculist.h> |
1bc144b62 net, rds, Replace... |
36 |
#include <linux/llist.h> |
08b48a1ed RDS/IB: Implement... |
37 |
|
0cb43965d RDS: split out co... |
38 |
#include "rds_single_path.h" |
f6df683f3 RDS: IB: Re-organ... |
39 |
#include "ib_mr.h" |
2eafa1746 net/rds: Handle O... |
40 |
#include "rds.h" |
f6df683f3 RDS: IB: Re-organ... |
41 42 |
struct workqueue_struct *rds_ib_mr_wq; |
2eafa1746 net/rds: Handle O... |
43 44 45 46 47 48 |
struct rds_ib_dereg_odp_mr { struct work_struct work; struct ib_mr *mr; }; static void rds_ib_odp_mr_worker(struct work_struct *work); |
08b48a1ed RDS/IB: Implement... |
49 |
|
08b48a1ed RDS/IB: Implement... |
50 51 52 53 |
static struct rds_ib_device *rds_ib_get_device(__be32 ipaddr) { struct rds_ib_device *rds_ibdev; struct rds_ib_ipaddr *i_ipaddr; |
ea819867b RDS/IB: protect t... |
54 55 |
rcu_read_lock(); list_for_each_entry_rcu(rds_ibdev, &rds_ib_devices, list) { |
764f2dd92 rds: rcu-ize rds_... |
56 |
list_for_each_entry_rcu(i_ipaddr, &rds_ibdev->ipaddr_list, list) { |
08b48a1ed RDS/IB: Implement... |
57 |
if (i_ipaddr->ipaddr == ipaddr) { |
50d61ff78 net, rds: convert... |
58 |
refcount_inc(&rds_ibdev->refcount); |
764f2dd92 rds: rcu-ize rds_... |
59 |
rcu_read_unlock(); |
08b48a1ed RDS/IB: Implement... |
60 61 62 |
return rds_ibdev; } } |
08b48a1ed RDS/IB: Implement... |
63 |
} |
ea819867b RDS/IB: protect t... |
64 |
rcu_read_unlock(); |
08b48a1ed RDS/IB: Implement... |
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
return NULL; } static int rds_ib_add_ipaddr(struct rds_ib_device *rds_ibdev, __be32 ipaddr) { struct rds_ib_ipaddr *i_ipaddr; i_ipaddr = kmalloc(sizeof *i_ipaddr, GFP_KERNEL); if (!i_ipaddr) return -ENOMEM; i_ipaddr->ipaddr = ipaddr; spin_lock_irq(&rds_ibdev->spinlock); |
764f2dd92 rds: rcu-ize rds_... |
80 |
list_add_tail_rcu(&i_ipaddr->list, &rds_ibdev->ipaddr_list); |
08b48a1ed RDS/IB: Implement... |
81 82 83 84 85 86 87 |
spin_unlock_irq(&rds_ibdev->spinlock); return 0; } static void rds_ib_remove_ipaddr(struct rds_ib_device *rds_ibdev, __be32 ipaddr) { |
4a81802b5 RDS/IB: Remove un... |
88 |
struct rds_ib_ipaddr *i_ipaddr; |
764f2dd92 rds: rcu-ize rds_... |
89 |
struct rds_ib_ipaddr *to_free = NULL; |
08b48a1ed RDS/IB: Implement... |
90 91 |
spin_lock_irq(&rds_ibdev->spinlock); |
764f2dd92 rds: rcu-ize rds_... |
92 |
list_for_each_entry_rcu(i_ipaddr, &rds_ibdev->ipaddr_list, list) { |
08b48a1ed RDS/IB: Implement... |
93 |
if (i_ipaddr->ipaddr == ipaddr) { |
764f2dd92 rds: rcu-ize rds_... |
94 95 |
list_del_rcu(&i_ipaddr->list); to_free = i_ipaddr; |
08b48a1ed RDS/IB: Implement... |
96 97 98 99 |
break; } } spin_unlock_irq(&rds_ibdev->spinlock); |
764f2dd92 rds: rcu-ize rds_... |
100 |
|
59fe46067 RDS: use kfree_rc... |
101 102 |
if (to_free) kfree_rcu(to_free, rcu); |
08b48a1ed RDS/IB: Implement... |
103 |
} |
eee2fa6ab rds: Changing IP ... |
104 105 |
int rds_ib_update_ipaddr(struct rds_ib_device *rds_ibdev, struct in6_addr *ipaddr) |
08b48a1ed RDS/IB: Implement... |
106 107 |
{ struct rds_ib_device *rds_ibdev_old; |
eee2fa6ab rds: Changing IP ... |
108 |
rds_ibdev_old = rds_ib_get_device(ipaddr->s6_addr32[3]); |
e1f475a73 RDS: don't update... |
109 |
if (!rds_ibdev_old) |
eee2fa6ab rds: Changing IP ... |
110 |
return rds_ib_add_ipaddr(rds_ibdev, ipaddr->s6_addr32[3]); |
e1f475a73 RDS: don't update... |
111 112 |
if (rds_ibdev_old != rds_ibdev) { |
eee2fa6ab rds: Changing IP ... |
113 |
rds_ib_remove_ipaddr(rds_ibdev_old, ipaddr->s6_addr32[3]); |
3e0249f9c RDS/IB: add refco... |
114 |
rds_ib_dev_put(rds_ibdev_old); |
eee2fa6ab rds: Changing IP ... |
115 |
return rds_ib_add_ipaddr(rds_ibdev, ipaddr->s6_addr32[3]); |
3e0249f9c RDS/IB: add refco... |
116 |
} |
e1f475a73 RDS: don't update... |
117 |
rds_ib_dev_put(rds_ibdev_old); |
08b48a1ed RDS/IB: Implement... |
118 |
|
e1f475a73 RDS: don't update... |
119 |
return 0; |
08b48a1ed RDS/IB: Implement... |
120 |
} |
745cbccac RDS: Rewrite conn... |
121 |
void rds_ib_add_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn) |
08b48a1ed RDS/IB: Implement... |
122 123 124 125 126 127 128 129 |
{ struct rds_ib_connection *ic = conn->c_transport_data; /* conn was previously on the nodev_conns_list */ spin_lock_irq(&ib_nodev_conns_lock); BUG_ON(list_empty(&ib_nodev_conns)); BUG_ON(list_empty(&ic->ib_node)); list_del(&ic->ib_node); |
08b48a1ed RDS/IB: Implement... |
130 |
|
aef3ea33e rds: spin_lock_ir... |
131 |
spin_lock(&rds_ibdev->spinlock); |
08b48a1ed RDS/IB: Implement... |
132 |
list_add_tail(&ic->ib_node, &rds_ibdev->conn_list); |
aef3ea33e rds: spin_lock_ir... |
133 |
spin_unlock(&rds_ibdev->spinlock); |
745cbccac RDS: Rewrite conn... |
134 |
spin_unlock_irq(&ib_nodev_conns_lock); |
08b48a1ed RDS/IB: Implement... |
135 136 |
ic->rds_ibdev = rds_ibdev; |
50d61ff78 net, rds: convert... |
137 |
refcount_inc(&rds_ibdev->refcount); |
08b48a1ed RDS/IB: Implement... |
138 |
} |
745cbccac RDS: Rewrite conn... |
139 |
void rds_ib_remove_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn) |
08b48a1ed RDS/IB: Implement... |
140 |
{ |
745cbccac RDS: Rewrite conn... |
141 |
struct rds_ib_connection *ic = conn->c_transport_data; |
08b48a1ed RDS/IB: Implement... |
142 |
|
745cbccac RDS: Rewrite conn... |
143 144 |
/* place conn on nodev_conns_list */ spin_lock(&ib_nodev_conns_lock); |
08b48a1ed RDS/IB: Implement... |
145 |
|
745cbccac RDS: Rewrite conn... |
146 147 148 149 150 151 152 153 154 155 |
spin_lock_irq(&rds_ibdev->spinlock); BUG_ON(list_empty(&ic->ib_node)); list_del(&ic->ib_node); spin_unlock_irq(&rds_ibdev->spinlock); list_add_tail(&ic->ib_node, &ib_nodev_conns); spin_unlock(&ib_nodev_conns_lock); ic->rds_ibdev = NULL; |
3e0249f9c RDS/IB: add refco... |
156 |
rds_ib_dev_put(rds_ibdev); |
08b48a1ed RDS/IB: Implement... |
157 |
} |
8aeb1ba66 RDS/IB: destroy c... |
158 |
void rds_ib_destroy_nodev_conns(void) |
08b48a1ed RDS/IB: Implement... |
159 160 161 162 163 |
{ struct rds_ib_connection *ic, *_ic; LIST_HEAD(tmp_list); /* avoid calling conn_destroy with irqs off */ |
8aeb1ba66 RDS/IB: destroy c... |
164 165 166 |
spin_lock_irq(&ib_nodev_conns_lock); list_splice(&ib_nodev_conns, &tmp_list); spin_unlock_irq(&ib_nodev_conns_lock); |
08b48a1ed RDS/IB: Implement... |
167 |
|
433d308dd RDS: Fix panic on... |
168 |
list_for_each_entry_safe(ic, _ic, &tmp_list, ib_node) |
08b48a1ed RDS/IB: Implement... |
169 |
rds_conn_destroy(ic->conn); |
08b48a1ed RDS/IB: Implement... |
170 |
} |
08b48a1ed RDS/IB: Implement... |
171 172 |
void rds_ib_get_mr_info(struct rds_ib_device *rds_ibdev, struct rds_info_rdma_connection *iinfo) { |
067665132 RDS: IB: split mr... |
173 |
struct rds_ib_mr_pool *pool_1m = rds_ibdev->mr_1m_pool; |
08b48a1ed RDS/IB: Implement... |
174 |
|
067665132 RDS: IB: split mr... |
175 |
iinfo->rdma_mr_max = pool_1m->max_items; |
07549ee21 RDMA/rds: Remove ... |
176 |
iinfo->rdma_mr_size = pool_1m->max_pages; |
08b48a1ed RDS/IB: Implement... |
177 |
} |
e65d4d963 rds: Remove IPv6 ... |
178 |
#if IS_ENABLED(CONFIG_IPV6) |
b7ff8b103 rds: Extend RDS A... |
179 180 181 182 183 184 |
void rds6_ib_get_mr_info(struct rds_ib_device *rds_ibdev, struct rds6_info_rdma_connection *iinfo6) { struct rds_ib_mr_pool *pool_1m = rds_ibdev->mr_1m_pool; iinfo6->rdma_mr_max = pool_1m->max_items; |
07549ee21 RDMA/rds: Remove ... |
185 |
iinfo6->rdma_mr_size = pool_1m->max_pages; |
b7ff8b103 rds: Extend RDS A... |
186 |
} |
e65d4d963 rds: Remove IPv6 ... |
187 |
#endif |
b7ff8b103 rds: Extend RDS A... |
188 |
|
f6df683f3 RDS: IB: Re-organ... |
189 |
struct rds_ib_mr *rds_ib_reuse_mr(struct rds_ib_mr_pool *pool) |
08b48a1ed RDS/IB: Implement... |
190 191 |
{ struct rds_ib_mr *ibmr = NULL; |
1bc144b62 net, rds, Replace... |
192 |
struct llist_node *ret; |
c9467447f net/rds: Get rid ... |
193 |
unsigned long flags; |
08b48a1ed RDS/IB: Implement... |
194 |
|
c9467447f net/rds: Get rid ... |
195 |
spin_lock_irqsave(&pool->clean_lock, flags); |
1bc144b62 net, rds, Replace... |
196 |
ret = llist_del_first(&pool->clean_list); |
c9467447f net/rds: Get rid ... |
197 |
spin_unlock_irqrestore(&pool->clean_lock, flags); |
db42753ad RDS: IB: add mr r... |
198 |
if (ret) { |
1bc144b62 net, rds, Replace... |
199 |
ibmr = llist_entry(ret, struct rds_ib_mr, llnode); |
db42753ad RDS: IB: add mr r... |
200 201 202 203 204 |
if (pool->pool_type == RDS_IB_MR_8K_POOL) rds_ib_stats_inc(s_ib_rdma_mr_8k_reused); else rds_ib_stats_inc(s_ib_rdma_mr_1m_reused); } |
08b48a1ed RDS/IB: Implement... |
205 206 207 |
return ibmr; } |
08b48a1ed RDS/IB: Implement... |
208 209 210 211 |
void rds_ib_sync_mr(void *trans_private, int direction) { struct rds_ib_mr *ibmr = trans_private; struct rds_ib_device *rds_ibdev = ibmr->device; |
2eafa1746 net/rds: Handle O... |
212 213 |
if (ibmr->odp) return; |
08b48a1ed RDS/IB: Implement... |
214 215 216 217 218 219 220 221 222 223 224 |
switch (direction) { case DMA_FROM_DEVICE: ib_dma_sync_sg_for_cpu(rds_ibdev->dev, ibmr->sg, ibmr->sg_dma_len, DMA_BIDIRECTIONAL); break; case DMA_TO_DEVICE: ib_dma_sync_sg_for_device(rds_ibdev->dev, ibmr->sg, ibmr->sg_dma_len, DMA_BIDIRECTIONAL); break; } } |
f6df683f3 RDS: IB: Re-organ... |
225 |
void __rds_ib_teardown_mr(struct rds_ib_mr *ibmr) |
08b48a1ed RDS/IB: Implement... |
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 |
{ struct rds_ib_device *rds_ibdev = ibmr->device; if (ibmr->sg_dma_len) { ib_dma_unmap_sg(rds_ibdev->dev, ibmr->sg, ibmr->sg_len, DMA_BIDIRECTIONAL); ibmr->sg_dma_len = 0; } /* Release the s/g list */ if (ibmr->sg_len) { unsigned int i; for (i = 0; i < ibmr->sg_len; ++i) { struct page *page = sg_page(&ibmr->sg[i]); /* FIXME we need a way to tell a r/w MR * from a r/o MR */ |
5c240fa2a RDS: Fix assertio... |
245 |
WARN_ON(!page->mapping && irqs_disabled()); |
08b48a1ed RDS/IB: Implement... |
246 247 248 249 250 251 252 253 254 |
set_page_dirty(page); put_page(page); } kfree(ibmr->sg); ibmr->sg = NULL; ibmr->sg_len = 0; } } |
f6df683f3 RDS: IB: Re-organ... |
255 |
void rds_ib_teardown_mr(struct rds_ib_mr *ibmr) |
08b48a1ed RDS/IB: Implement... |
256 257 258 259 260 |
{ unsigned int pinned = ibmr->sg_len; __rds_ib_teardown_mr(ibmr); if (pinned) { |
26139dc1d RDS: IB: use alre... |
261 |
struct rds_ib_mr_pool *pool = ibmr->pool; |
08b48a1ed RDS/IB: Implement... |
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 |
atomic_sub(pinned, &pool->free_pinned); } } static inline unsigned int rds_ib_flush_goal(struct rds_ib_mr_pool *pool, int free_all) { unsigned int item_count; item_count = atomic_read(&pool->item_count); if (free_all) return item_count; return 0; } /* |
1bc144b62 net, rds, Replace... |
279 |
* given an llist of mrs, put them all into the list_head for more processing |
6fa70da60 rds: recycle FMRs... |
280 |
*/ |
6116c2030 RDS: fix fmr pool... |
281 282 |
static unsigned int llist_append_to_list(struct llist_head *llist, struct list_head *list) |
6fa70da60 rds: recycle FMRs... |
283 284 |
{ struct rds_ib_mr *ibmr; |
1bc144b62 net, rds, Replace... |
285 286 |
struct llist_node *node; struct llist_node *next; |
6116c2030 RDS: fix fmr pool... |
287 |
unsigned int count = 0; |
1bc144b62 net, rds, Replace... |
288 289 290 291 292 |
node = llist_del_all(llist); while (node) { next = node->next; ibmr = llist_entry(node, struct rds_ib_mr, llnode); |
6fa70da60 rds: recycle FMRs... |
293 |
list_add_tail(&ibmr->unmap_list, list); |
1bc144b62 net, rds, Replace... |
294 |
node = next; |
6116c2030 RDS: fix fmr pool... |
295 |
count++; |
6fa70da60 rds: recycle FMRs... |
296 |
} |
6116c2030 RDS: fix fmr pool... |
297 |
return count; |
6fa70da60 rds: recycle FMRs... |
298 299 300 |
} /* |
1bc144b62 net, rds, Replace... |
301 302 303 |
* this takes a list head of mrs and turns it into linked llist nodes * of clusters. Each cluster has linked llist nodes of * MR_CLUSTER_SIZE mrs that are ready for reuse. |
6fa70da60 rds: recycle FMRs... |
304 |
*/ |
c9467447f net/rds: Get rid ... |
305 |
static void list_to_llist_nodes(struct list_head *list, |
1bc144b62 net, rds, Replace... |
306 307 |
struct llist_node **nodes_head, struct llist_node **nodes_tail) |
6fa70da60 rds: recycle FMRs... |
308 309 |
{ struct rds_ib_mr *ibmr; |
1bc144b62 net, rds, Replace... |
310 311 |
struct llist_node *cur = NULL; struct llist_node **next = nodes_head; |
6fa70da60 rds: recycle FMRs... |
312 313 |
list_for_each_entry(ibmr, list, unmap_list) { |
1bc144b62 net, rds, Replace... |
314 315 316 |
cur = &ibmr->llnode; *next = cur; next = &cur->next; |
6fa70da60 rds: recycle FMRs... |
317 |
} |
1bc144b62 net, rds, Replace... |
318 319 |
*next = NULL; *nodes_tail = cur; |
6fa70da60 rds: recycle FMRs... |
320 321 322 |
} /* |
08b48a1ed RDS/IB: Implement... |
323 324 325 326 327 |
* Flush our pool of MRs. * At a minimum, all currently unused MRs are unmapped. * If the number of MRs allocated exceeds the limit, we also try * to free as many MRs as needed to get back to this limit. */ |
f6df683f3 RDS: IB: Re-organ... |
328 329 |
int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool, int free_all, struct rds_ib_mr **ibmr_ret) |
08b48a1ed RDS/IB: Implement... |
330 |
{ |
490ea5967 RDS: IB: move FMR... |
331 |
struct rds_ib_mr *ibmr; |
1bc144b62 net, rds, Replace... |
332 333 |
struct llist_node *clean_nodes; struct llist_node *clean_tail; |
08b48a1ed RDS/IB: Implement... |
334 |
LIST_HEAD(unmap_list); |
08b48a1ed RDS/IB: Implement... |
335 |
unsigned long unpinned = 0; |
6116c2030 RDS: fix fmr pool... |
336 |
unsigned int nfreed = 0, dirty_to_clean = 0, free_goal; |
08b48a1ed RDS/IB: Implement... |
337 |
|
067665132 RDS: IB: split mr... |
338 339 340 341 |
if (pool->pool_type == RDS_IB_MR_8K_POOL) rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_flush); else rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_flush); |
08b48a1ed RDS/IB: Implement... |
342 |
|
6fa70da60 rds: recycle FMRs... |
343 344 |
if (ibmr_ret) { DEFINE_WAIT(wait); |
067665132 RDS: IB: split mr... |
345 |
while (!mutex_trylock(&pool->flush_lock)) { |
f6df683f3 RDS: IB: Re-organ... |
346 |
ibmr = rds_ib_reuse_mr(pool); |
6fa70da60 rds: recycle FMRs... |
347 348 349 350 351 352 353 354 |
if (ibmr) { *ibmr_ret = ibmr; finish_wait(&pool->flush_wait, &wait); goto out_nolock; } prepare_to_wait(&pool->flush_wait, &wait, TASK_UNINTERRUPTIBLE); |
1bc144b62 net, rds, Replace... |
355 |
if (llist_empty(&pool->clean_list)) |
6fa70da60 rds: recycle FMRs... |
356 |
schedule(); |
f6df683f3 RDS: IB: Re-organ... |
357 |
ibmr = rds_ib_reuse_mr(pool); |
6fa70da60 rds: recycle FMRs... |
358 359 360 361 362 363 364 365 366 367 368 |
if (ibmr) { *ibmr_ret = ibmr; finish_wait(&pool->flush_wait, &wait); goto out_nolock; } } finish_wait(&pool->flush_wait, &wait); } else mutex_lock(&pool->flush_lock); if (ibmr_ret) { |
f6df683f3 RDS: IB: Re-organ... |
369 |
ibmr = rds_ib_reuse_mr(pool); |
6fa70da60 rds: recycle FMRs... |
370 371 372 373 374 |
if (ibmr) { *ibmr_ret = ibmr; goto out; } } |
08b48a1ed RDS/IB: Implement... |
375 |
|
08b48a1ed RDS/IB: Implement... |
376 |
/* Get the list of all MRs to be dropped. Ordering matters - |
6fa70da60 rds: recycle FMRs... |
377 378 |
* we want to put drop_list ahead of free_list. */ |
6116c2030 RDS: fix fmr pool... |
379 380 |
dirty_to_clean = llist_append_to_list(&pool->drop_list, &unmap_list); dirty_to_clean += llist_append_to_list(&pool->free_list, &unmap_list); |
c9467447f net/rds: Get rid ... |
381 382 383 384 |
if (free_all) { unsigned long flags; spin_lock_irqsave(&pool->clean_lock, flags); |
1bc144b62 net, rds, Replace... |
385 |
llist_append_to_list(&pool->clean_list, &unmap_list); |
c9467447f net/rds: Get rid ... |
386 387 |
spin_unlock_irqrestore(&pool->clean_lock, flags); } |
08b48a1ed RDS/IB: Implement... |
388 389 390 391 392 |
free_goal = rds_ib_flush_goal(pool, free_all); if (list_empty(&unmap_list)) goto out; |
07549ee21 RDMA/rds: Remove ... |
393 |
rds_ib_unreg_frmr(&unmap_list, &nfreed, &unpinned, free_goal); |
08b48a1ed RDS/IB: Implement... |
394 |
|
6fa70da60 rds: recycle FMRs... |
395 |
if (!list_empty(&unmap_list)) { |
c9467447f net/rds: Get rid ... |
396 397 398 |
unsigned long flags; list_to_llist_nodes(&unmap_list, &clean_nodes, &clean_tail); |
85cb92878 net: rds: fix mem... |
399 |
if (ibmr_ret) { |
1bc144b62 net, rds, Replace... |
400 |
*ibmr_ret = llist_entry(clean_nodes, struct rds_ib_mr, llnode); |
85cb92878 net: rds: fix mem... |
401 402 |
clean_nodes = clean_nodes->next; } |
1bc144b62 net, rds, Replace... |
403 |
/* more than one entry in llist nodes */ |
c9467447f net/rds: Get rid ... |
404 405 |
if (clean_nodes) { spin_lock_irqsave(&pool->clean_lock, flags); |
85cb92878 net: rds: fix mem... |
406 407 |
llist_add_batch(clean_nodes, clean_tail, &pool->clean_list); |
c9467447f net/rds: Get rid ... |
408 409 |
spin_unlock_irqrestore(&pool->clean_lock, flags); } |
6fa70da60 rds: recycle FMRs... |
410 |
} |
08b48a1ed RDS/IB: Implement... |
411 412 |
atomic_sub(unpinned, &pool->free_pinned); |
6116c2030 RDS: fix fmr pool... |
413 |
atomic_sub(dirty_to_clean, &pool->dirty_count); |
08b48a1ed RDS/IB: Implement... |
414 415 416 417 |
atomic_sub(nfreed, &pool->item_count); out: mutex_unlock(&pool->flush_lock); |
6fa70da60 rds: recycle FMRs... |
418 419 420 |
if (waitqueue_active(&pool->flush_wait)) wake_up(&pool->flush_wait); out_nolock: |
490ea5967 RDS: IB: move FMR... |
421 422 423 424 425 426 427 |
return 0; } struct rds_ib_mr *rds_ib_try_reuse_ibmr(struct rds_ib_mr_pool *pool) { struct rds_ib_mr *ibmr = NULL; int iter = 0; |
490ea5967 RDS: IB: move FMR... |
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 |
while (1) { ibmr = rds_ib_reuse_mr(pool); if (ibmr) return ibmr; if (atomic_inc_return(&pool->item_count) <= pool->max_items) break; atomic_dec(&pool->item_count); if (++iter > 2) { if (pool->pool_type == RDS_IB_MR_8K_POOL) rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_depleted); else rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_depleted); |
aea01a223 net/rds: Fix NULL... |
443 |
break; |
490ea5967 RDS: IB: move FMR... |
444 445 446 447 448 449 450 451 452 453 454 455 |
} /* We do have some empty MRs. Flush them out. */ if (pool->pool_type == RDS_IB_MR_8K_POOL) rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_wait); else rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_wait); rds_ib_flush_mr_pool(pool, 0, &ibmr); if (ibmr) return ibmr; } |
aea01a223 net/rds: Fix NULL... |
456 |
return NULL; |
08b48a1ed RDS/IB: Implement... |
457 458 459 460 |
} static void rds_ib_mr_pool_flush_worker(struct work_struct *work) { |
7a0ff5dbd RDS: use delayed ... |
461 |
struct rds_ib_mr_pool *pool = container_of(work, struct rds_ib_mr_pool, flush_worker.work); |
08b48a1ed RDS/IB: Implement... |
462 |
|
6fa70da60 rds: recycle FMRs... |
463 |
rds_ib_flush_mr_pool(pool, 0, NULL); |
08b48a1ed RDS/IB: Implement... |
464 465 466 467 468 |
} void rds_ib_free_mr(void *trans_private, int invalidate) { struct rds_ib_mr *ibmr = trans_private; |
26139dc1d RDS: IB: use alre... |
469 |
struct rds_ib_mr_pool *pool = ibmr->pool; |
08b48a1ed RDS/IB: Implement... |
470 |
struct rds_ib_device *rds_ibdev = ibmr->device; |
08b48a1ed RDS/IB: Implement... |
471 472 473 |
rdsdebug("RDS/IB: free_mr nents %u ", ibmr->sg_len); |
2eafa1746 net/rds: Handle O... |
474 475 476 477 478 479 480 481 482 |
if (ibmr->odp) { /* A MR created and marked as use_once. We use delayed work, * because there is a change that we are in interrupt and can't * call to ib_dereg_mr() directly. */ INIT_DELAYED_WORK(&ibmr->work, rds_ib_odp_mr_worker); queue_delayed_work(rds_ib_mr_wq, &ibmr->work, 0); return; } |
08b48a1ed RDS/IB: Implement... |
483 |
/* Return it to the pool's free list */ |
07549ee21 RDMA/rds: Remove ... |
484 |
rds_ib_free_frmr_list(ibmr); |
08b48a1ed RDS/IB: Implement... |
485 486 487 |
atomic_add(ibmr->sg_len, &pool->free_pinned); atomic_inc(&pool->dirty_count); |
08b48a1ed RDS/IB: Implement... |
488 489 |
/* If we've pinned too many pages, request a flush */ |
f64f9e719 net: Move && and ... |
490 |
if (atomic_read(&pool->free_pinned) >= pool->max_free_pinned || |
ef5217a6e RDS: flush the FM... |
491 |
atomic_read(&pool->dirty_count) >= pool->max_items / 5) |
f6df683f3 RDS: IB: Re-organ... |
492 |
queue_delayed_work(rds_ib_mr_wq, &pool->flush_worker, 10); |
08b48a1ed RDS/IB: Implement... |
493 494 495 |
if (invalidate) { if (likely(!in_interrupt())) { |
6fa70da60 rds: recycle FMRs... |
496 |
rds_ib_flush_mr_pool(pool, 0, NULL); |
08b48a1ed RDS/IB: Implement... |
497 498 |
} else { /* We get here if the user created a MR marked |
ad1d7dc0d RDS: push FMR poo... |
499 500 |
* as use_once and invalidate at the same time. */ |
f6df683f3 RDS: IB: Re-organ... |
501 |
queue_delayed_work(rds_ib_mr_wq, |
ad1d7dc0d RDS: push FMR poo... |
502 |
&pool->flush_worker, 10); |
08b48a1ed RDS/IB: Implement... |
503 504 |
} } |
3e0249f9c RDS/IB: add refco... |
505 506 |
rds_ib_dev_put(rds_ibdev); |
08b48a1ed RDS/IB: Implement... |
507 508 509 510 511 |
} void rds_ib_flush_mrs(void) { struct rds_ib_device *rds_ibdev; |
ea819867b RDS/IB: protect t... |
512 |
down_read(&rds_ib_devices_lock); |
08b48a1ed RDS/IB: Implement... |
513 |
list_for_each_entry(rds_ibdev, &rds_ib_devices, list) { |
067665132 RDS: IB: split mr... |
514 515 |
if (rds_ibdev->mr_8k_pool) rds_ib_flush_mr_pool(rds_ibdev->mr_8k_pool, 0, NULL); |
08b48a1ed RDS/IB: Implement... |
516 |
|
067665132 RDS: IB: split mr... |
517 518 |
if (rds_ibdev->mr_1m_pool) rds_ib_flush_mr_pool(rds_ibdev->mr_1m_pool, 0, NULL); |
08b48a1ed RDS/IB: Implement... |
519 |
} |
ea819867b RDS/IB: protect t... |
520 |
up_read(&rds_ib_devices_lock); |
08b48a1ed RDS/IB: Implement... |
521 |
} |
2eafa1746 net/rds: Handle O... |
522 523 524 525 526 527 |
u32 rds_ib_get_lkey(void *trans_private) { struct rds_ib_mr *ibmr = trans_private; return ibmr->u.mr->lkey; } |
08b48a1ed RDS/IB: Implement... |
528 |
void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents, |
9e630bcb7 RDS: RDMA: Fix th... |
529 |
struct rds_sock *rs, u32 *key_ret, |
2eafa1746 net/rds: Handle O... |
530 531 |
struct rds_connection *conn, u64 start, u64 length, int need_odp) |
08b48a1ed RDS/IB: Implement... |
532 533 534 |
{ struct rds_ib_device *rds_ibdev; struct rds_ib_mr *ibmr = NULL; |
9e630bcb7 RDS: RDMA: Fix th... |
535 |
struct rds_ib_connection *ic = NULL; |
08b48a1ed RDS/IB: Implement... |
536 |
int ret; |
eee2fa6ab rds: Changing IP ... |
537 |
rds_ibdev = rds_ib_get_device(rs->rs_bound_addr.s6_addr32[3]); |
08b48a1ed RDS/IB: Implement... |
538 539 540 541 |
if (!rds_ibdev) { ret = -ENODEV; goto out; } |
2eafa1746 net/rds: Handle O... |
542 543 544 545 546 547 |
if (need_odp == ODP_ZEROBASED || need_odp == ODP_VIRTUAL) { u64 virt_addr = need_odp == ODP_ZEROBASED ? 0 : start; int access_flags = (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_ON_DEMAND); |
b2dfc6765 net/rds: Use pref... |
548 |
struct ib_sge sge = {}; |
2eafa1746 net/rds: Handle O... |
549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 |
struct ib_mr *ib_mr; if (!rds_ibdev->odp_capable) { ret = -EOPNOTSUPP; goto out; } ib_mr = ib_reg_user_mr(rds_ibdev->pd, start, length, virt_addr, access_flags); if (IS_ERR(ib_mr)) { rdsdebug("rds_ib_get_user_mr returned %d ", IS_ERR(ib_mr)); ret = PTR_ERR(ib_mr); goto out; } if (key_ret) *key_ret = ib_mr->rkey; ibmr = kzalloc(sizeof(*ibmr), GFP_KERNEL); if (!ibmr) { ib_dereg_mr(ib_mr); ret = -ENOMEM; goto out; } ibmr->u.mr = ib_mr; ibmr->odp = 1; |
b2dfc6765 net/rds: Use pref... |
577 578 579 580 581 582 583 584 |
sge.addr = virt_addr; sge.length = length; sge.lkey = ib_mr->lkey; ib_advise_mr(rds_ibdev->pd, IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE, IB_UVERBS_ADVISE_MR_FLAG_FLUSH, &sge, 1); |
2eafa1746 net/rds: Handle O... |
585 586 |
return ibmr; } |
9e630bcb7 RDS: RDMA: Fix th... |
587 588 |
if (conn) ic = conn->c_transport_data; |
067665132 RDS: IB: split mr... |
589 |
if (!rds_ibdev->mr_8k_pool || !rds_ibdev->mr_1m_pool) { |
08b48a1ed RDS/IB: Implement... |
590 591 592 |
ret = -ENODEV; goto out; } |
07549ee21 RDMA/rds: Remove ... |
593 |
ibmr = rds_ib_reg_frmr(rds_ibdev, ic, sg, nents, key_ret); |
9e630bcb7 RDS: RDMA: Fix th... |
594 595 |
if (IS_ERR(ibmr)) { ret = PTR_ERR(ibmr); |
490ea5967 RDS: IB: move FMR... |
596 597 |
pr_warn("RDS/IB: rds_ib_get_mr failed (errno=%d) ", ret); |
9e630bcb7 RDS: RDMA: Fix th... |
598 599 600 |
} else { return ibmr; } |
490ea5967 RDS: IB: move FMR... |
601 |
|
9e630bcb7 RDS: RDMA: Fix th... |
602 |
out: |
3e0249f9c RDS/IB: add refco... |
603 604 |
if (rds_ibdev) rds_ib_dev_put(rds_ibdev); |
490ea5967 RDS: IB: move FMR... |
605 |
|
9e630bcb7 RDS: RDMA: Fix th... |
606 |
return ERR_PTR(ret); |
08b48a1ed RDS/IB: Implement... |
607 |
} |
6fa70da60 rds: recycle FMRs... |
608 |
|
f6df683f3 RDS: IB: Re-organ... |
609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 |
void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *pool) { cancel_delayed_work_sync(&pool->flush_worker); rds_ib_flush_mr_pool(pool, 1, NULL); WARN_ON(atomic_read(&pool->item_count)); WARN_ON(atomic_read(&pool->free_pinned)); kfree(pool); } struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_ibdev, int pool_type) { struct rds_ib_mr_pool *pool; pool = kzalloc(sizeof(*pool), GFP_KERNEL); if (!pool) return ERR_PTR(-ENOMEM); pool->pool_type = pool_type; init_llist_head(&pool->free_list); init_llist_head(&pool->drop_list); init_llist_head(&pool->clean_list); |
c9467447f net/rds: Get rid ... |
631 |
spin_lock_init(&pool->clean_lock); |
f6df683f3 RDS: IB: Re-organ... |
632 633 634 635 636 637 |
mutex_init(&pool->flush_lock); init_waitqueue_head(&pool->flush_wait); INIT_DELAYED_WORK(&pool->flush_worker, rds_ib_mr_pool_flush_worker); if (pool_type == RDS_IB_MR_1M_POOL) { /* +1 allows for unaligned MRs */ |
07549ee21 RDMA/rds: Remove ... |
638 |
pool->max_pages = RDS_MR_1M_MSG_SIZE + 1; |
b1fb67fa5 RDS: IB: Initiali... |
639 |
pool->max_items = rds_ibdev->max_1m_mrs; |
f6df683f3 RDS: IB: Re-organ... |
640 641 |
} else { /* pool_type == RDS_IB_MR_8K_POOL */ |
07549ee21 RDMA/rds: Remove ... |
642 |
pool->max_pages = RDS_MR_8K_MSG_SIZE + 1; |
b1fb67fa5 RDS: IB: Initiali... |
643 |
pool->max_items = rds_ibdev->max_8k_mrs; |
f6df683f3 RDS: IB: Re-organ... |
644 |
} |
07549ee21 RDMA/rds: Remove ... |
645 |
pool->max_free_pinned = pool->max_items * pool->max_pages / 4; |
f6df683f3 RDS: IB: Re-organ... |
646 647 648 649 650 651 652 |
pool->max_items_soft = rds_ibdev->max_mrs * 3 / 4; return pool; } int rds_ib_mr_init(void) { |
231edca97 RDS: IB: Remove d... |
653 |
rds_ib_mr_wq = alloc_workqueue("rds_mr_flushd", WQ_MEM_RECLAIM, 0); |
f6df683f3 RDS: IB: Re-organ... |
654 655 656 657 658 659 660 661 662 663 664 665 666 |
if (!rds_ib_mr_wq) return -ENOMEM; return 0; } /* By the time this is called all the IB devices should have been torn down and * had their pools freed. As each pool is freed its work struct is waited on, * so the pool flushing work queue should be idle by the time we get here. */ void rds_ib_mr_exit(void) { destroy_workqueue(rds_ib_mr_wq); } |
2eafa1746 net/rds: Handle O... |
667 668 669 670 671 672 673 674 675 |
static void rds_ib_odp_mr_worker(struct work_struct *work) { struct rds_ib_mr *ibmr; ibmr = container_of(work, struct rds_ib_mr, work.work); ib_dereg_mr(ibmr->u.mr); kfree(ibmr); } |