Blame view
net/smc/smc_core.c
18.6 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
0cfdd8f92 smc: connection a... |
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 |
/* * Shared Memory Communications over RDMA (SMC-R) and RoCE * * Basic Transport Functions exploiting Infiniband API * * Copyright IBM Corp. 2016 * * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> */ #include <linux/socket.h> #include <linux/if_vlan.h> #include <linux/random.h> #include <linux/workqueue.h> #include <net/tcp.h> #include <net/sock.h> #include <rdma/ib_verbs.h> #include "smc.h" #include "smc_clc.h" #include "smc_core.h" #include "smc_ib.h" |
f38ba179c smc: work request... |
24 |
#include "smc_wr.h" |
9bf9abead smc: link layer c... |
25 |
#include "smc_llc.h" |
5f08318f6 smc: connection d... |
26 |
#include "smc_cdc.h" |
b38d73247 smc: socket closi... |
27 |
#include "smc_close.h" |
0cfdd8f92 smc: connection a... |
28 |
|
5bc11ddbd net/smc: longer d... |
29 30 31 |
#define SMC_LGR_NUM_INCR 256 #define SMC_LGR_FREE_DELAY_SERV (600 * HZ) #define SMC_LGR_FREE_DELAY_CLNT (SMC_LGR_FREE_DELAY_SERV + 10) |
0cfdd8f92 smc: connection a... |
32 |
|
9bf9abead smc: link layer c... |
33 |
static u32 smc_lgr_num; /* unique link group number */ |
0cfdd8f92 smc: connection a... |
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 |
/* Register connection's alert token in our lookup structure. * To use rbtrees we have to implement our own insert core. * Requires @conns_lock * @smc connection to register * Returns 0 on success, != otherwise. */ static void smc_lgr_add_alert_token(struct smc_connection *conn) { struct rb_node **link, *parent = NULL; u32 token = conn->alert_token_local; link = &conn->lgr->conns_all.rb_node; while (*link) { struct smc_connection *cur = rb_entry(*link, struct smc_connection, alert_node); parent = *link; if (cur->alert_token_local > token) link = &parent->rb_left; else link = &parent->rb_right; } /* Put the new node there */ rb_link_node(&conn->alert_node, parent, link); rb_insert_color(&conn->alert_node, &conn->lgr->conns_all); } /* Register connection in link group by assigning an alert token * registered in a search tree. * Requires @conns_lock * Note that '0' is a reserved value and not assigned. */ static void smc_lgr_register_conn(struct smc_connection *conn) { struct smc_sock *smc = container_of(conn, struct smc_sock, conn); static atomic_t nexttoken = ATOMIC_INIT(0); /* find a new alert_token_local value not yet used by some connection * in this link group */ sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */ while (!conn->alert_token_local) { conn->alert_token_local = atomic_inc_return(&nexttoken); if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr)) conn->alert_token_local = 0; } smc_lgr_add_alert_token(conn); conn->lgr->conns_num++; } /* Unregister connection and reset the alert token of the given connection< */ static void __smc_lgr_unregister_conn(struct smc_connection *conn) { struct smc_sock *smc = container_of(conn, struct smc_sock, conn); struct smc_link_group *lgr = conn->lgr; rb_erase(&conn->alert_node, &lgr->conns_all); lgr->conns_num--; conn->alert_token_local = 0; conn->lgr = NULL; sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */ } /* Unregister connection and trigger lgr freeing if applicable */ static void smc_lgr_unregister_conn(struct smc_connection *conn) { struct smc_link_group *lgr = conn->lgr; int reduced = 0; write_lock_bh(&lgr->conns_lock); if (conn->alert_token_local) { reduced = 1; __smc_lgr_unregister_conn(conn); } write_unlock_bh(&lgr->conns_lock); |
5bc11ddbd net/smc: longer d... |
111 112 113 114 115 116 117 118 119 |
if (!reduced || lgr->conns_num) return; /* client link group creation always follows the server link group * creation. For client use a somewhat higher removal delay time, * otherwise there is a risk of out-of-sync link groups. */ mod_delayed_work(system_wq, &lgr->free_work, lgr->role == SMC_CLNT ? SMC_LGR_FREE_DELAY_CLNT : SMC_LGR_FREE_DELAY_SERV); |
0cfdd8f92 smc: connection a... |
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 |
} static void smc_lgr_free_work(struct work_struct *work) { struct smc_link_group *lgr = container_of(to_delayed_work(work), struct smc_link_group, free_work); bool conns; spin_lock_bh(&smc_lgr_list.lock); read_lock_bh(&lgr->conns_lock); conns = RB_EMPTY_ROOT(&lgr->conns_all); read_unlock_bh(&lgr->conns_lock); if (!conns) { /* number of lgr connections is no longer zero */ spin_unlock_bh(&smc_lgr_list.lock); return; } list_del_init(&lgr->list); /* remove from smc_lgr_list */ spin_unlock_bh(&smc_lgr_list.lock); smc_lgr_free(lgr); } /* create a new SMC link group */ static int smc_lgr_create(struct smc_sock *smc, __be32 peer_in_addr, struct smc_ib_device *smcibdev, u8 ibport, char *peer_systemid, unsigned short vlan_id) { struct smc_link_group *lgr; struct smc_link *lnk; u8 rndvec[3]; int rc = 0; |
cd6851f30 smc: remote memor... |
151 |
int i; |
0cfdd8f92 smc: connection a... |
152 153 154 155 156 157 158 159 160 161 162 |
lgr = kzalloc(sizeof(*lgr), GFP_KERNEL); if (!lgr) { rc = -ENOMEM; goto out; } lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT; lgr->sync_err = false; lgr->daddr = peer_in_addr; memcpy(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN); lgr->vlan_id = vlan_id; |
cd6851f30 smc: remote memor... |
163 164 165 166 167 168 |
rwlock_init(&lgr->sndbufs_lock); rwlock_init(&lgr->rmbs_lock); for (i = 0; i < SMC_RMBE_SIZES; i++) { INIT_LIST_HEAD(&lgr->sndbufs[i]); INIT_LIST_HEAD(&lgr->rmbs[i]); } |
9bf9abead smc: link layer c... |
169 170 |
smc_lgr_num += SMC_LGR_NUM_INCR; memcpy(&lgr->id, (u8 *)&smc_lgr_num, SMC_LGR_ID_SIZE); |
0cfdd8f92 smc: connection a... |
171 172 173 174 175 |
INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work); lgr->conns_all = RB_ROOT; lnk = &lgr->lnk[SMC_SINGLE_LINK]; /* initialize link */ |
953a64ca3 net/smc: use link... |
176 |
lnk->link_id = SMC_SINGLE_LINK; |
0cfdd8f92 smc: connection a... |
177 178 179 |
lnk->smcibdev = smcibdev; lnk->ibport = ibport; lnk->path_mtu = smcibdev->pattr[ibport - 1].active_mtu; |
bd4ad5771 smc: initialize I... |
180 181 |
if (!smcibdev->initialized) smc_ib_setup_per_ibdev(smcibdev); |
0cfdd8f92 smc: connection a... |
182 183 |
get_random_bytes(rndvec, sizeof(rndvec)); lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + (rndvec[2] << 16); |
f38ba179c smc: work request... |
184 185 186 |
rc = smc_wr_alloc_link_mem(lnk); if (rc) goto free_lgr; |
bd4ad5771 smc: initialize I... |
187 188 189 190 191 192 193 194 195 |
rc = smc_ib_create_protection_domain(lnk); if (rc) goto free_link_mem; rc = smc_ib_create_queue_pair(lnk); if (rc) goto dealloc_pd; rc = smc_wr_create_link(lnk); if (rc) goto destroy_qp; |
9bf9abead smc: link layer c... |
196 197 |
init_completion(&lnk->llc_confirm); init_completion(&lnk->llc_confirm_resp); |
0cfdd8f92 smc: connection a... |
198 199 200 201 202 203 |
smc->conn.lgr = lgr; rwlock_init(&lgr->conns_lock); spin_lock_bh(&smc_lgr_list.lock); list_add(&lgr->list, &smc_lgr_list.list); spin_unlock_bh(&smc_lgr_list.lock); |
f38ba179c smc: work request... |
204 |
return 0; |
bd4ad5771 smc: initialize I... |
205 206 207 208 209 210 |
destroy_qp: smc_ib_destroy_queue_pair(lnk); dealloc_pd: smc_ib_dealloc_protection_domain(lnk); free_link_mem: smc_wr_free_link_mem(lnk); |
f38ba179c smc: work request... |
211 212 |
free_lgr: kfree(lgr); |
0cfdd8f92 smc: connection a... |
213 214 215 |
out: return rc; } |
3e034725c net/smc: common f... |
216 |
static void smc_buf_unuse(struct smc_connection *conn) |
cd6851f30 smc: remote memor... |
217 218 219 220 221 |
{ if (conn->sndbuf_desc) { conn->sndbuf_desc->used = 0; conn->sndbuf_size = 0; } |
cd6851f30 smc: remote memor... |
222 |
if (conn->rmb_desc) { |
897e1c245 net/smc: use sepa... |
223 |
conn->rmb_desc->reused = true; |
cd6851f30 smc: remote memor... |
224 225 226 227 |
conn->rmb_desc->used = 0; conn->rmbe_size = 0; } } |
0cfdd8f92 smc: connection a... |
228 229 230 231 232 233 234 |
/* remove a finished connection from its link group */ void smc_conn_free(struct smc_connection *conn) { struct smc_link_group *lgr = conn->lgr; if (!lgr) return; |
5f08318f6 smc: connection d... |
235 |
smc_cdc_tx_dismiss_slots(conn); |
0cfdd8f92 smc: connection a... |
236 |
smc_lgr_unregister_conn(conn); |
3e034725c net/smc: common f... |
237 |
smc_buf_unuse(conn); |
0cfdd8f92 smc: connection a... |
238 239 240 241 242 |
} static void smc_link_clear(struct smc_link *lnk) { lnk->peer_qpn = 0; |
bd4ad5771 smc: initialize I... |
243 |
smc_ib_modify_qp_reset(lnk); |
f38ba179c smc: work request... |
244 |
smc_wr_free_link(lnk); |
bd4ad5771 smc: initialize I... |
245 246 |
smc_ib_destroy_queue_pair(lnk); smc_ib_dealloc_protection_domain(lnk); |
f38ba179c smc: work request... |
247 |
smc_wr_free_link_mem(lnk); |
0cfdd8f92 smc: connection a... |
248 |
} |
3e034725c net/smc: common f... |
249 250 |
static void smc_buf_free(struct smc_buf_desc *buf_desc, struct smc_link *lnk, bool is_rmb) |
cd6851f30 smc: remote memor... |
251 |
{ |
3e034725c net/smc: common f... |
252 253 254 255 256 257 258 259 260 |
if (is_rmb) { if (buf_desc->mr_rx[SMC_SINGLE_LINK]) smc_ib_put_memory_region( buf_desc->mr_rx[SMC_SINGLE_LINK]); smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, DMA_FROM_DEVICE); } else { smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, DMA_TO_DEVICE); |
cd6851f30 smc: remote memor... |
261 |
} |
3e034725c net/smc: common f... |
262 263 264 265 |
sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]); if (buf_desc->cpu_addr) free_pages((unsigned long)buf_desc->cpu_addr, buf_desc->order); kfree(buf_desc); |
cd6851f30 smc: remote memor... |
266 |
} |
3e034725c net/smc: common f... |
267 |
static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb) |
cd6851f30 smc: remote memor... |
268 |
{ |
bd4ad5771 smc: initialize I... |
269 |
struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; |
3e034725c net/smc: common f... |
270 271 |
struct smc_buf_desc *buf_desc, *bf_desc; struct list_head *buf_list; |
cd6851f30 smc: remote memor... |
272 273 274 |
int i; for (i = 0; i < SMC_RMBE_SIZES; i++) { |
3e034725c net/smc: common f... |
275 276 277 278 279 |
if (is_rmb) buf_list = &lgr->rmbs[i]; else buf_list = &lgr->sndbufs[i]; list_for_each_entry_safe(buf_desc, bf_desc, buf_list, |
cd6851f30 smc: remote memor... |
280 |
list) { |
3e034725c net/smc: common f... |
281 282 |
list_del(&buf_desc->list); smc_buf_free(buf_desc, lnk, is_rmb); |
cd6851f30 smc: remote memor... |
283 284 285 |
} } } |
3e034725c net/smc: common f... |
286 287 288 289 290 291 292 |
static void smc_lgr_free_bufs(struct smc_link_group *lgr) { /* free send buffers */ __smc_lgr_free_bufs(lgr, false); /* free rmbs */ __smc_lgr_free_bufs(lgr, true); } |
0cfdd8f92 smc: connection a... |
293 294 295 |
/* remove a link group */ void smc_lgr_free(struct smc_link_group *lgr) { |
3e034725c net/smc: common f... |
296 |
smc_lgr_free_bufs(lgr); |
0cfdd8f92 smc: connection a... |
297 298 299 300 301 302 303 304 |
smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]); kfree(lgr); } /* terminate linkgroup abnormally */ void smc_lgr_terminate(struct smc_link_group *lgr) { struct smc_connection *conn; |
b38d73247 smc: socket closi... |
305 |
struct smc_sock *smc; |
0cfdd8f92 smc: connection a... |
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 |
struct rb_node *node; spin_lock_bh(&smc_lgr_list.lock); if (list_empty(&lgr->list)) { /* termination already triggered */ spin_unlock_bh(&smc_lgr_list.lock); return; } /* do not use this link group for new connections */ list_del_init(&lgr->list); spin_unlock_bh(&smc_lgr_list.lock); write_lock_bh(&lgr->conns_lock); node = rb_first(&lgr->conns_all); while (node) { conn = rb_entry(node, struct smc_connection, alert_node); |
b38d73247 smc: socket closi... |
322 323 |
smc = container_of(conn, struct smc_sock, conn); sock_hold(&smc->sk); |
0cfdd8f92 smc: connection a... |
324 |
__smc_lgr_unregister_conn(conn); |
46c28dbd4 net/smc: no socke... |
325 |
schedule_work(&conn->close_work); |
b38d73247 smc: socket closi... |
326 |
sock_put(&smc->sk); |
0cfdd8f92 smc: connection a... |
327 328 329 |
node = rb_first(&lgr->conns_all); } write_unlock_bh(&lgr->conns_lock); |
0cfdd8f92 smc: connection a... |
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 |
} /* Determine vlan of internal TCP socket. * @vlan_id: address to store the determined vlan id into */ static int smc_vlan_by_tcpsk(struct socket *clcsock, unsigned short *vlan_id) { struct dst_entry *dst = sk_dst_get(clcsock->sk); int rc = 0; *vlan_id = 0; if (!dst) { rc = -ENOTCONN; goto out; } if (!dst->dev) { rc = -ENODEV; goto out_rel; } if (is_vlan_dev(dst->dev)) *vlan_id = vlan_dev_vlan_id(dst->dev); out_rel: dst_release(dst); out: return rc; } /* determine the link gid matching the vlan id of the link group */ static int smc_link_determine_gid(struct smc_link_group *lgr) { struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; struct ib_gid_attr gattr; union ib_gid gid; int i; if (!lgr->vlan_id) { lnk->gid = lnk->smcibdev->gid[lnk->ibport - 1]; return 0; } for (i = 0; i < lnk->smcibdev->pattr[lnk->ibport - 1].gid_tbl_len; i++) { if (ib_query_gid(lnk->smcibdev->ibdev, lnk->ibport, i, &gid, &gattr)) continue; if (gattr.ndev && (vlan_dev_vlan_id(gattr.ndev) == lgr->vlan_id)) { lnk->gid = gid; return 0; } } return -ENODEV; } /* create a new SMC connection (and a new link group if necessary) */ int smc_conn_create(struct smc_sock *smc, __be32 peer_in_addr, struct smc_ib_device *smcibdev, u8 ibport, struct smc_clc_msg_local *lcl, int srv_first_contact) { struct smc_connection *conn = &smc->conn; struct smc_link_group *lgr; unsigned short vlan_id; enum smc_lgr_role role; int local_contact = SMC_FIRST_CONTACT; int rc = 0; role = smc->listen_smc ? SMC_SERV : SMC_CLNT; rc = smc_vlan_by_tcpsk(smc->clcsock, &vlan_id); if (rc) return rc; if ((role == SMC_CLNT) && srv_first_contact) /* create new link group as well */ goto create; /* determine if an existing link group can be reused */ spin_lock_bh(&smc_lgr_list.lock); list_for_each_entry(lgr, &smc_lgr_list.list, list) { write_lock_bh(&lgr->conns_lock); if (!memcmp(lgr->peer_systemid, lcl->id_for_peer, SMC_SYSTEMID_LEN) && !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid, SMC_GID_SIZE) && !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac, sizeof(lcl->mac)) && !lgr->sync_err && (lgr->role == role) && |
cd6851f30 smc: remote memor... |
419 420 421 |
(lgr->vlan_id == vlan_id) && ((role == SMC_CLNT) || (lgr->conns_num < SMC_RMBS_PER_LGR_MAX))) { |
0cfdd8f92 smc: connection a... |
422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 |
/* link group found */ local_contact = SMC_REUSE_CONTACT; conn->lgr = lgr; smc_lgr_register_conn(conn); /* add smc conn to lgr */ write_unlock_bh(&lgr->conns_lock); break; } write_unlock_bh(&lgr->conns_lock); } spin_unlock_bh(&smc_lgr_list.lock); if (role == SMC_CLNT && !srv_first_contact && (local_contact == SMC_FIRST_CONTACT)) { /* Server reuses a link group, but Client wants to start * a new one * send out_of_sync decline, reason synchr. error */ return -ENOLINK; } create: if (local_contact == SMC_FIRST_CONTACT) { rc = smc_lgr_create(smc, peer_in_addr, smcibdev, ibport, lcl->id_for_peer, vlan_id); if (rc) goto out; smc_lgr_register_conn(conn); /* add smc conn to lgr */ rc = smc_link_determine_gid(conn->lgr); } |
5f08318f6 smc: connection d... |
451 452 453 454 455 |
conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE; conn->local_tx_ctrl.len = sizeof(struct smc_cdc_msg); #ifndef KERNEL_HAS_ATOMIC64 spin_lock_init(&conn->acurs_lock); #endif |
0cfdd8f92 smc: connection a... |
456 457 458 459 |
out: return rc ? rc : local_contact; } |
cd6851f30 smc: remote memor... |
460 |
|
3e034725c net/smc: common f... |
461 462 |
/* try to reuse a sndbuf or rmb description slot for a certain * buffer size; if not available, return NULL |
cd6851f30 smc: remote memor... |
463 464 |
*/ static inline |
3e034725c net/smc: common f... |
465 466 467 468 |
struct smc_buf_desc *smc_buf_get_slot(struct smc_link_group *lgr, int compressed_bufsize, rwlock_t *lock, struct list_head *buf_list) |
cd6851f30 smc: remote memor... |
469 |
{ |
3e034725c net/smc: common f... |
470 |
struct smc_buf_desc *buf_slot; |
cd6851f30 smc: remote memor... |
471 |
|
3e034725c net/smc: common f... |
472 473 474 475 476 |
read_lock_bh(lock); list_for_each_entry(buf_slot, buf_list, list) { if (cmpxchg(&buf_slot->used, 0, 1) == 0) { read_unlock_bh(lock); return buf_slot; |
cd6851f30 smc: remote memor... |
477 478 |
} } |
3e034725c net/smc: common f... |
479 |
read_unlock_bh(lock); |
cd6851f30 smc: remote memor... |
480 481 |
return NULL; } |
952310ccf smc: receive data... |
482 483 484 485 486 487 488 489 |
/* one of the conditions for announcing a receiver's current window size is * that it "results in a minimum increase in the window size of 10% of the * receive buffer space" [RFC7609] */ static inline int smc_rmb_wnd_update_limit(int rmbe_size) { return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2); } |
b33982c3a net/smc: cleanup ... |
490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 |
static struct smc_buf_desc *smc_new_buf_create(struct smc_link_group *lgr, bool is_rmb, int bufsize) { struct smc_buf_desc *buf_desc; struct smc_link *lnk; int rc; /* try to alloc a new buffer */ buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL); if (!buf_desc) return ERR_PTR(-ENOMEM); buf_desc->cpu_addr = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_ZERO, get_order(bufsize)); if (!buf_desc->cpu_addr) { kfree(buf_desc); return ERR_PTR(-EAGAIN); } buf_desc->order = get_order(bufsize); /* build the sg table from the pages */ lnk = &lgr->lnk[SMC_SINGLE_LINK]; rc = sg_alloc_table(&buf_desc->sgt[SMC_SINGLE_LINK], 1, GFP_KERNEL); if (rc) { smc_buf_free(buf_desc, lnk, is_rmb); return ERR_PTR(rc); } sg_set_buf(buf_desc->sgt[SMC_SINGLE_LINK].sgl, buf_desc->cpu_addr, bufsize); /* map sg table to DMA address */ rc = smc_ib_buf_map_sg(lnk->smcibdev, buf_desc, is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE); /* SMC protocol depends on mapping to one DMA address only */ if (rc != 1) { smc_buf_free(buf_desc, lnk, is_rmb); return ERR_PTR(-EAGAIN); } /* create a new memory region for the RMB */ if (is_rmb) { rc = smc_ib_get_memory_region(lnk->roce_pd, IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE, buf_desc); if (rc) { smc_buf_free(buf_desc, lnk, is_rmb); return ERR_PTR(rc); } } return buf_desc; } |
3e034725c net/smc: common f... |
547 |
static int __smc_buf_create(struct smc_sock *smc, bool is_rmb) |
cd6851f30 smc: remote memor... |
548 549 550 |
{ struct smc_connection *conn = &smc->conn; struct smc_link_group *lgr = conn->lgr; |
3e034725c net/smc: common f... |
551 552 |
struct smc_buf_desc *buf_desc = NULL; struct list_head *buf_list; |
c45abf31e net/smc: shorten ... |
553 |
int bufsize, bufsize_short; |
3e034725c net/smc: common f... |
554 555 |
int sk_buf_size; rwlock_t *lock; |
cd6851f30 smc: remote memor... |
556 |
|
3e034725c net/smc: common f... |
557 558 559 560 561 562 |
if (is_rmb) /* use socket recv buffer size (w/o overhead) as start value */ sk_buf_size = smc->sk.sk_rcvbuf / 2; else /* use socket send buffer size (w/o overhead) as start value */ sk_buf_size = smc->sk.sk_sndbuf / 2; |
4d0d1bc65 net/smc: use sk_r... |
563 |
for (bufsize_short = smc_compress_bufsize(sk_buf_size); |
c45abf31e net/smc: shorten ... |
564 |
bufsize_short >= 0; bufsize_short--) { |
9d8fb6173 net/smc: introduc... |
565 |
|
3e034725c net/smc: common f... |
566 567 568 569 570 571 |
if (is_rmb) { lock = &lgr->rmbs_lock; buf_list = &lgr->rmbs[bufsize_short]; } else { lock = &lgr->sndbufs_lock; buf_list = &lgr->sndbufs[bufsize_short]; |
9d8fb6173 net/smc: introduc... |
572 |
} |
c45abf31e net/smc: shorten ... |
573 |
bufsize = smc_uncompress_bufsize(bufsize_short); |
a3fe3d01b net/smc: introduc... |
574 575 |
if ((1 << get_order(bufsize)) > SG_MAX_SINGLE_ALLOC) continue; |
3e034725c net/smc: common f... |
576 577 578 579 |
/* check for reusable slot in the link group */ buf_desc = smc_buf_get_slot(lgr, bufsize_short, lock, buf_list); if (buf_desc) { memset(buf_desc->cpu_addr, 0, bufsize); |
cd6851f30 smc: remote memor... |
580 581 |
break; /* found reusable slot */ } |
a3fe3d01b net/smc: introduc... |
582 |
|
b33982c3a net/smc: cleanup ... |
583 584 585 586 |
buf_desc = smc_new_buf_create(lgr, is_rmb, bufsize); if (PTR_ERR(buf_desc) == -ENOMEM) break; if (IS_ERR(buf_desc)) |
a3fe3d01b net/smc: introduc... |
587 |
continue; |
897e1c245 net/smc: use sepa... |
588 |
|
3e034725c net/smc: common f... |
589 590 591 592 593 |
buf_desc->used = 1; write_lock_bh(lock); list_add(&buf_desc->list, buf_list); write_unlock_bh(lock); break; /* found */ |
cd6851f30 smc: remote memor... |
594 |
} |
3e034725c net/smc: common f... |
595 |
|
b33982c3a net/smc: cleanup ... |
596 |
if (IS_ERR(buf_desc)) |
3e034725c net/smc: common f... |
597 598 599 600 |
return -ENOMEM; if (is_rmb) { conn->rmb_desc = buf_desc; |
c45abf31e net/smc: shorten ... |
601 602 603 |
conn->rmbe_size = bufsize; conn->rmbe_size_short = bufsize_short; smc->sk.sk_rcvbuf = bufsize * 2; |
5f08318f6 smc: connection d... |
604 |
atomic_set(&conn->bytes_to_rcv, 0); |
c45abf31e net/smc: shorten ... |
605 |
conn->rmbe_update_limit = smc_rmb_wnd_update_limit(bufsize); |
cd6851f30 smc: remote memor... |
606 |
} else { |
3e034725c net/smc: common f... |
607 608 609 610 |
conn->sndbuf_desc = buf_desc; conn->sndbuf_size = bufsize; smc->sk.sk_sndbuf = bufsize * 2; atomic_set(&conn->sndbuf_space, bufsize); |
cd6851f30 smc: remote memor... |
611 |
} |
3e034725c net/smc: common f... |
612 613 |
return 0; } |
10428dd83 net/smc: synchron... |
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 |
void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn) { struct smc_link_group *lgr = conn->lgr; smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev, conn->sndbuf_desc, DMA_TO_DEVICE); } void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn) { struct smc_link_group *lgr = conn->lgr; smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev, conn->sndbuf_desc, DMA_TO_DEVICE); } void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn) { struct smc_link_group *lgr = conn->lgr; smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev, conn->rmb_desc, DMA_FROM_DEVICE); } void smc_rmb_sync_sg_for_device(struct smc_connection *conn) { struct smc_link_group *lgr = conn->lgr; smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev, conn->rmb_desc, DMA_FROM_DEVICE); } |
3e034725c net/smc: common f... |
645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 |
/* create the send and receive buffer for an SMC socket; * receive buffers are called RMBs; * (even though the SMC protocol allows more than one RMB-element per RMB, * the Linux implementation uses just one RMB-element per RMB, i.e. uses an * extra RMB for every connection in a link group */ int smc_buf_create(struct smc_sock *smc) { int rc; /* create send buffer */ rc = __smc_buf_create(smc, false); if (rc) return rc; /* create rmb */ rc = __smc_buf_create(smc, true); if (rc) smc_buf_free(smc->conn.sndbuf_desc, &smc->conn.lgr->lnk[SMC_SINGLE_LINK], false); return rc; |
cd6851f30 smc: remote memor... |
665 |
} |
bd4ad5771 smc: initialize I... |
666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 |
static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr) { int i; for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) { if (!test_and_set_bit(i, lgr->rtokens_used_mask)) return i; } return -ENOSPC; } /* save rkey and dma_addr received from peer during clc handshake */ int smc_rmb_rtoken_handling(struct smc_connection *conn, struct smc_clc_msg_accept_confirm *clc) { u64 dma_addr = be64_to_cpu(clc->rmb_dma_addr); struct smc_link_group *lgr = conn->lgr; u32 rkey = ntohl(clc->rmb_rkey); int i; for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) { if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) && |
263eec9b2 smc: switch to us... |
689 |
(lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) && |
bd4ad5771 smc: initialize I... |
690 691 692 693 694 695 696 697 698 699 700 701 |
test_bit(i, lgr->rtokens_used_mask)) { conn->rtoken_idx = i; return 0; } } conn->rtoken_idx = smc_rmb_reserve_rtoken_idx(lgr); if (conn->rtoken_idx < 0) return conn->rtoken_idx; lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].rkey = rkey; lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].dma_addr = dma_addr; return 0; } |