Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit d521b63b27e3a397e0ef7ca86b6e813861083c83

Authored by Andy Grover 2009-10-30 16:51:57 +0800

Committed by David S. Miller 2009-10-31 06:06:39 +0800

Exists in master and in 7 other branches

RDS/IB+IW: Move recv processing to a tasklet

Move receive processing from event handler to a tasklet.
This should help prevent hangcheck timer from going off
when RDS is under heavy load.

Signed-off-by: Andy Grover <andy.grover@oracle.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

Showing 6 changed files with 52 additions and 12 deletions Inline Diff

net/rds/ib.h
net/rds/ib_cm.c
net/rds/ib_recv.c
net/rds/iw.h
net/rds/iw_cm.c
net/rds/iw_recv.c

net/rds/ib.h

Diff comments View file @ d521b63

 #ifndef _RDS_IB_H
 #define _RDS_IB_H
 #include <rdma/ib_verbs.h>
 #include <rdma/rdma_cm.h>
 #include "rds.h"
 #include "rdma_transport.h"
 #define RDS_FMR_SIZE			256
 #define RDS_FMR_POOL_SIZE		4096
 #define RDS_IB_MAX_SGE			8
 #define RDS_IB_RECV_SGE 		2
 #define RDS_IB_DEFAULT_RECV_WR		1024
 #define RDS_IB_DEFAULT_SEND_WR		256
 #define RDS_IB_DEFAULT_RETRY_COUNT	2
 #define RDS_IB_SUPPORTED_PROTOCOLS	0x00000003	/* minor versions supported */
 extern struct list_head rds_ib_devices;
 /*
  * IB posts RDS_FRAG_SIZE fragments of pages to the receive queues to
  * try and minimize the amount of memory tied up both the device and
  * socket receive queues.
  */
 /* page offset of the final full frag that fits in the page */
 #define RDS_PAGE_LAST_OFF (((PAGE_SIZE  / RDS_FRAG_SIZE) - 1) * RDS_FRAG_SIZE)
 struct rds_page_frag {
 	struct list_head	f_item;
 	struct page		*f_page;
 	unsigned long		f_offset;
 	dma_addr_t 		f_mapped;
 };
 struct rds_ib_incoming {
 	struct list_head	ii_frags;
 	struct rds_incoming	ii_inc;
 };
 struct rds_ib_connect_private {
 	/* Add new fields at the end, and don't permute existing fields. */
 	__be32			dp_saddr;
 	__be32			dp_daddr;
 	u8			dp_protocol_major;
 	u8			dp_protocol_minor;
 	__be16			dp_protocol_minor_mask; /* bitmask */
 	__be32			dp_reserved1;
 	__be64			dp_ack_seq;
 	__be32			dp_credit;		/* non-zero enables flow ctl */
 };
 struct rds_ib_send_work {
 	struct rds_message	*s_rm;
 	struct rds_rdma_op	*s_op;
 	struct ib_send_wr	s_wr;
 	struct ib_sge		s_sge[RDS_IB_MAX_SGE];
 	unsigned long		s_queued;
 };
 struct rds_ib_recv_work {
 	struct rds_ib_incoming 	*r_ibinc;
 	struct rds_page_frag	*r_frag;
 	struct ib_recv_wr	r_wr;
 	struct ib_sge		r_sge[2];
 };
 struct rds_ib_work_ring {
 	u32		w_nr;
 	u32		w_alloc_ptr;
 	u32		w_alloc_ctr;
 	u32		w_free_ptr;
 	atomic_t	w_free_ctr;
 };
 struct rds_ib_device;
 struct rds_ib_connection {
 	struct list_head	ib_node;
 	struct rds_ib_device	*rds_ibdev;
 	struct rds_connection	*conn;
 	/* alphabet soup, IBTA style */
 	struct rdma_cm_id	*i_cm_id;
 	struct ib_pd		*i_pd;
 	struct ib_mr		*i_mr;
 	struct ib_cq		*i_send_cq;
 	struct ib_cq		*i_recv_cq;
 	/* tx */
 	struct rds_ib_work_ring	i_send_ring;
 	struct rds_message	*i_rm;
 	struct rds_header	*i_send_hdrs;
 	u64			i_send_hdrs_dma;
 	struct rds_ib_send_work *i_sends;
 	/* rx */
+	struct tasklet_struct	i_recv_tasklet;
 	struct mutex		i_recv_mutex;
 	struct rds_ib_work_ring	i_recv_ring;
 	struct rds_ib_incoming	*i_ibinc;
 	u32			i_recv_data_rem;
 	struct rds_header	*i_recv_hdrs;
 	u64			i_recv_hdrs_dma;
 	struct rds_ib_recv_work *i_recvs;
 	struct rds_page_frag	i_frag;
 	u64			i_ack_recv;	/* last ACK received */
 	/* sending acks */
 	unsigned long		i_ack_flags;
 #ifdef KERNEL_HAS_ATOMIC64
 	atomic64_t		i_ack_next;	/* next ACK to send */
 #else
 	spinlock_t		i_ack_lock;	/* protect i_ack_next */
 	u64			i_ack_next;	/* next ACK to send */
 #endif
 	struct rds_header	*i_ack;
 	struct ib_send_wr	i_ack_wr;
 	struct ib_sge		i_ack_sge;
 	u64			i_ack_dma;
 	unsigned long		i_ack_queued;
 	/* Flow control related information
 	 *
 	 * Our algorithm uses a pair variables that we need to access
 	 * atomically - one for the send credits, and one posted
 	 * recv credits we need to transfer to remote.
 	 * Rather than protect them using a slow spinlock, we put both into
 	 * a single atomic_t and update it using cmpxchg
 	 */
 	atomic_t		i_credits;
 	/* Protocol version specific information */
 	unsigned int		i_flowctl:1;	/* enable/disable flow ctl */
 	/* Batched completions */
 	unsigned int		i_unsignaled_wrs;
 	long			i_unsignaled_bytes;
 };
 /* This assumes that atomic_t is at least 32 bits */
 #define IB_GET_SEND_CREDITS(v)	((v) & 0xffff)
 #define IB_GET_POST_CREDITS(v)	((v) >> 16)
 #define IB_SET_SEND_CREDITS(v)	((v) & 0xffff)
 #define IB_SET_POST_CREDITS(v)	((v) << 16)
 struct rds_ib_ipaddr {
 	struct list_head	list;
 	__be32			ipaddr;
 };
 struct rds_ib_device {
 	struct list_head	list;
 	struct list_head	ipaddr_list;
 	struct list_head	conn_list;
 	struct ib_device	*dev;
 	struct ib_pd		*pd;
 	struct ib_mr		*mr;
 	struct rds_ib_mr_pool	*mr_pool;
 	unsigned int		fmr_max_remaps;
 	unsigned int		max_fmrs;
 	int			max_sge;
 	unsigned int		max_wrs;
 	spinlock_t		spinlock;	/* protect the above */
 };
 /* bits for i_ack_flags */
 #define IB_ACK_IN_FLIGHT	0
 #define IB_ACK_REQUESTED	1
 /* Magic WR_ID for ACKs */
 #define RDS_IB_ACK_WR_ID	(~(u64) 0)
 struct rds_ib_statistics {
 	uint64_t	s_ib_connect_raced;
 	uint64_t	s_ib_listen_closed_stale;
 	uint64_t	s_ib_tx_cq_call;
 	uint64_t	s_ib_tx_cq_event;
 	uint64_t	s_ib_tx_ring_full;
 	uint64_t	s_ib_tx_throttle;
 	uint64_t	s_ib_tx_sg_mapping_failure;
 	uint64_t	s_ib_tx_stalled;
 	uint64_t	s_ib_tx_credit_updates;
 	uint64_t	s_ib_rx_cq_call;
 	uint64_t	s_ib_rx_cq_event;
 	uint64_t	s_ib_rx_ring_empty;
 	uint64_t	s_ib_rx_refill_from_cq;
 	uint64_t	s_ib_rx_refill_from_thread;
 	uint64_t	s_ib_rx_alloc_limit;
 	uint64_t	s_ib_rx_credit_updates;
 	uint64_t	s_ib_ack_sent;
 	uint64_t	s_ib_ack_send_failure;
 	uint64_t	s_ib_ack_send_delayed;
 	uint64_t	s_ib_ack_send_piggybacked;
 	uint64_t	s_ib_ack_received;
 	uint64_t	s_ib_rdma_mr_alloc;
 	uint64_t	s_ib_rdma_mr_free;
 	uint64_t	s_ib_rdma_mr_used;
 	uint64_t	s_ib_rdma_mr_pool_flush;
 	uint64_t	s_ib_rdma_mr_pool_wait;
 	uint64_t	s_ib_rdma_mr_pool_depleted;
 };
 extern struct workqueue_struct *rds_ib_wq;
 /*
  * Fake ib_dma_sync_sg_for_{cpu,device} as long as ib_verbs.h
  * doesn't define it.
  */
 static inline void rds_ib_dma_sync_sg_for_cpu(struct ib_device *dev,
 		struct scatterlist *sg, unsigned int sg_dma_len, int direction)
 {
 	unsigned int i;
 	for (i = 0; i < sg_dma_len; ++i) {
 		ib_dma_sync_single_for_cpu(dev,
 				ib_sg_dma_address(dev, &sg[i]),
 				ib_sg_dma_len(dev, &sg[i]),
 				direction);
 	}
 }
 #define ib_dma_sync_sg_for_cpu	rds_ib_dma_sync_sg_for_cpu
 static inline void rds_ib_dma_sync_sg_for_device(struct ib_device *dev,
 		struct scatterlist *sg, unsigned int sg_dma_len, int direction)
 {
 	unsigned int i;
 	for (i = 0; i < sg_dma_len; ++i) {
 		ib_dma_sync_single_for_device(dev,
 				ib_sg_dma_address(dev, &sg[i]),
 				ib_sg_dma_len(dev, &sg[i]),
 				direction);
 	}
 }
 #define ib_dma_sync_sg_for_device	rds_ib_dma_sync_sg_for_device
 /* ib.c */
 extern struct rds_transport rds_ib_transport;
 extern void rds_ib_add_one(struct ib_device *device);
 extern void rds_ib_remove_one(struct ib_device *device);
 extern struct ib_client rds_ib_client;
 extern unsigned int fmr_pool_size;
 extern unsigned int fmr_message_size;
 extern unsigned int rds_ib_retry_count;
 extern spinlock_t ib_nodev_conns_lock;
 extern struct list_head ib_nodev_conns;
 /* ib_cm.c */
 int rds_ib_conn_alloc(struct rds_connection *conn, gfp_t gfp);
 void rds_ib_conn_free(void *arg);
 int rds_ib_conn_connect(struct rds_connection *conn);
 void rds_ib_conn_shutdown(struct rds_connection *conn);
 void rds_ib_state_change(struct sock *sk);
 int __init rds_ib_listen_init(void);
 void rds_ib_listen_stop(void);
 void __rds_ib_conn_error(struct rds_connection *conn, const char *, ...);
 int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
 			     struct rdma_cm_event *event);
 int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id);
 void rds_ib_cm_connect_complete(struct rds_connection *conn,
 				struct rdma_cm_event *event);
 #define rds_ib_conn_error(conn, fmt...) \
 	__rds_ib_conn_error(conn, KERN_WARNING "RDS/IB: " fmt)
 /* ib_rdma.c */
 int rds_ib_update_ipaddr(struct rds_ib_device *rds_ibdev, __be32 ipaddr);
 void rds_ib_add_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn);
 void rds_ib_remove_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn);
 void __rds_ib_destroy_conns(struct list_head *list, spinlock_t *list_lock);
 static inline void rds_ib_destroy_nodev_conns(void)
 {
 	__rds_ib_destroy_conns(&ib_nodev_conns, &ib_nodev_conns_lock);
 }
 static inline void rds_ib_destroy_conns(struct rds_ib_device *rds_ibdev)
 {
 	__rds_ib_destroy_conns(&rds_ibdev->conn_list, &rds_ibdev->spinlock);
 }
 struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *);
 void rds_ib_get_mr_info(struct rds_ib_device *rds_ibdev, struct rds_info_rdma_connection *iinfo);
 void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *);
 void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents,
 		    struct rds_sock *rs, u32 *key_ret);
 void rds_ib_sync_mr(void *trans_private, int dir);
 void rds_ib_free_mr(void *trans_private, int invalidate);
 void rds_ib_flush_mrs(void);
 /* ib_recv.c */
 int __init rds_ib_recv_init(void);
 void rds_ib_recv_exit(void);
 int rds_ib_recv(struct rds_connection *conn);
 int rds_ib_recv_refill(struct rds_connection *conn, gfp_t kptr_gfp,
 		       gfp_t page_gfp, int prefill);
 void rds_ib_inc_purge(struct rds_incoming *inc);
 void rds_ib_inc_free(struct rds_incoming *inc);
 int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iovec *iov,
 			     size_t size);
 void rds_ib_recv_cq_comp_handler(struct ib_cq *cq, void *context);
+void rds_ib_recv_tasklet_fn(unsigned long data);
 void rds_ib_recv_init_ring(struct rds_ib_connection *ic);
 void rds_ib_recv_clear_ring(struct rds_ib_connection *ic);
 void rds_ib_recv_init_ack(struct rds_ib_connection *ic);
 void rds_ib_attempt_ack(struct rds_ib_connection *ic);
 void rds_ib_ack_send_complete(struct rds_ib_connection *ic);
 u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic);
 /* ib_ring.c */
 void rds_ib_ring_init(struct rds_ib_work_ring *ring, u32 nr);
 void rds_ib_ring_resize(struct rds_ib_work_ring *ring, u32 nr);
 u32 rds_ib_ring_alloc(struct rds_ib_work_ring *ring, u32 val, u32 *pos);
 void rds_ib_ring_free(struct rds_ib_work_ring *ring, u32 val);
 void rds_ib_ring_unalloc(struct rds_ib_work_ring *ring, u32 val);
 int rds_ib_ring_empty(struct rds_ib_work_ring *ring);
 int rds_ib_ring_low(struct rds_ib_work_ring *ring);
 u32 rds_ib_ring_oldest(struct rds_ib_work_ring *ring);
 u32 rds_ib_ring_completed(struct rds_ib_work_ring *ring, u32 wr_id, u32 oldest);
 extern wait_queue_head_t rds_ib_ring_empty_wait;
 /* ib_send.c */
 void rds_ib_xmit_complete(struct rds_connection *conn);
 int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm,
 		unsigned int hdr_off, unsigned int sg, unsigned int off);
 void rds_ib_send_cq_comp_handler(struct ib_cq *cq, void *context);
 void rds_ib_send_init_ring(struct rds_ib_connection *ic);
 void rds_ib_send_clear_ring(struct rds_ib_connection *ic);
 int rds_ib_xmit_rdma(struct rds_connection *conn, struct rds_rdma_op *op);
 void rds_ib_send_add_credits(struct rds_connection *conn, unsigned int credits);
 void rds_ib_advertise_credits(struct rds_connection *conn, unsigned int posted);
 int rds_ib_send_grab_credits(struct rds_ib_connection *ic, u32 wanted,
 			     u32 *adv_credits, int need_posted, int max_posted);
 /* ib_stats.c */
 DECLARE_PER_CPU(struct rds_ib_statistics, rds_ib_stats);
 #define rds_ib_stats_inc(member) rds_stats_inc_which(rds_ib_stats, member)
 unsigned int rds_ib_stats_info_copy(struct rds_info_iterator *iter,
 				    unsigned int avail);
 /* ib_sysctl.c */
 int __init rds_ib_sysctl_init(void);
 void rds_ib_sysctl_exit(void);
 extern unsigned long rds_ib_sysctl_max_send_wr;
 extern unsigned long rds_ib_sysctl_max_recv_wr;
 extern unsigned long rds_ib_sysctl_max_unsig_wrs;
 extern unsigned long rds_ib_sysctl_max_unsig_bytes;
 extern unsigned long rds_ib_sysctl_max_recv_allocation;
 extern unsigned int rds_ib_sysctl_flow_control;
 extern ctl_table rds_ib_sysctl_table[];
 /*
  * Helper functions for getting/setting the header and data SGEs in
  * RDS packets (not RDMA)
  *
  * From version 3.1 onwards, header is in front of data in the sge.
  */
 static inline struct ib_sge *
 rds_ib_header_sge(struct rds_ib_connection *ic, struct ib_sge *sge)
 {
 	if (ic->conn->c_version > RDS_PROTOCOL_3_0)
 		return &sge[0];
 	else
 		return &sge[1];
 }
 static inline struct ib_sge *
 rds_ib_data_sge(struct rds_ib_connection *ic, struct ib_sge *sge)
 {
 	if (ic->conn->c_version > RDS_PROTOCOL_3_0)
 		return &sge[1];
 	else
 		return &sge[0];
 }
 #endif

net/rds/ib_cm.c

Diff comments View file @ d521b63

 /*
  * Copyright (c) 2006 Oracle.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
  * General Public License (GPL) Version 2, available from the file
  * COPYING in the main directory of this source tree, or the
  * OpenIB.org BSD license below:
  *
  *     Redistribution and use in source and binary forms, with or
  *     without modification, are permitted provided that the following
  *     conditions are met:
  *
  *      - Redistributions of source code must retain the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer.
  *
  *      - Redistributions in binary form must reproduce the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer in the documentation and/or other materials
  *        provided with the distribution.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  *
  */
 #include <linux/kernel.h>
 #include <linux/in.h>
 #include <linux/vmalloc.h>
 #include "rds.h"
 #include "ib.h"
 /*
  * Set the selected protocol version
  */
 static void rds_ib_set_protocol(struct rds_connection *conn, unsigned int version)
 {
 	conn->c_version = version;
 }
 /*
  * Set up flow control
  */
 static void rds_ib_set_flow_control(struct rds_connection *conn, u32 credits)
 {
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	if (rds_ib_sysctl_flow_control && credits != 0) {
 		/* We're doing flow control */
 		ic->i_flowctl = 1;
 		rds_ib_send_add_credits(conn, credits);
 	} else {
 		ic->i_flowctl = 0;
 	}
 }
 /*
  * Tune RNR behavior. Without flow control, we use a rather
  * low timeout, but not the absolute minimum - this should
  * be tunable.
  *
  * We already set the RNR retry count to 7 (which is the
  * smallest infinite number :-) above.
  * If flow control is off, we want to change this back to 0
  * so that we learn quickly when our credit accounting is
  * buggy.
  *
  * Caller passes in a qp_attr pointer - don't waste stack spacv
  * by allocation this twice.
  */
 static void
 rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr)
 {
 	int ret;
 	attr->min_rnr_timer = IB_RNR_TIMER_000_32;
 	ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);
 	if (ret)
 		printk(KERN_NOTICE "ib_modify_qp(IB_QP_MIN_RNR_TIMER): err=%d\n", -ret);
 }
 /*
  * Connection established.
  * We get here for both outgoing and incoming connection.
  */
 void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_event *event)
 {
 	const struct rds_ib_connect_private *dp = NULL;
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	struct rds_ib_device *rds_ibdev;
 	struct ib_qp_attr qp_attr;
 	int err;
 	if (event->param.conn.private_data_len >= sizeof(*dp)) {
 		dp = event->param.conn.private_data;
 		/* make sure it isn't empty data */
 		if (dp->dp_protocol_major) {
 			rds_ib_set_protocol(conn,
 				RDS_PROTOCOL(dp->dp_protocol_major,
 				dp->dp_protocol_minor));
 			rds_ib_set_flow_control(conn, be32_to_cpu(dp->dp_credit));
 		}
 	}
 	printk(KERN_NOTICE "RDS/IB: connected to %pI4 version %u.%u%s\n",
 			&conn->c_faddr,
 			RDS_PROTOCOL_MAJOR(conn->c_version),
 			RDS_PROTOCOL_MINOR(conn->c_version),
 			ic->i_flowctl ? ", flow control" : "");
 	/*
 	 * Init rings and fill recv. this needs to wait until protocol negotiation
 	 * is complete, since ring layout is different from 3.0 to 3.1.
 	 */
 	rds_ib_send_init_ring(ic);
 	rds_ib_recv_init_ring(ic);
 	/* Post receive buffers - as a side effect, this will update
 	 * the posted credit count. */
 	rds_ib_recv_refill(conn, GFP_KERNEL, GFP_HIGHUSER, 1);
 	/* Tune RNR behavior */
 	rds_ib_tune_rnr(ic, &qp_attr);
 	qp_attr.qp_state = IB_QPS_RTS;
 	err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);
 	if (err)
 		printk(KERN_NOTICE "ib_modify_qp(IB_QP_STATE, RTS): err=%d\n", err);
 	/* update ib_device with this local ipaddr & conn */
 	rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rds_ib_client);
 	err = rds_ib_update_ipaddr(rds_ibdev, conn->c_laddr);
 	if (err)
 		printk(KERN_ERR "rds_ib_update_ipaddr failed (%d)\n", err);
 	rds_ib_add_conn(rds_ibdev, conn);
 	/* If the peer gave us the last packet it saw, process this as if
 	 * we had received a regular ACK. */
 	if (dp && dp->dp_ack_seq)
 		rds_send_drop_acked(conn, be64_to_cpu(dp->dp_ack_seq), NULL);
 	rds_connect_complete(conn);
 }
 static void rds_ib_cm_fill_conn_param(struct rds_connection *conn,
 			struct rdma_conn_param *conn_param,
 			struct rds_ib_connect_private *dp,
 			u32 protocol_version)
 {
 	memset(conn_param, 0, sizeof(struct rdma_conn_param));
 	/* XXX tune these? */
 	conn_param->responder_resources = 1;
 	conn_param->initiator_depth = 1;
 	conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7);
 	conn_param->rnr_retry_count = 7;
 	if (dp) {
 		struct rds_ib_connection *ic = conn->c_transport_data;
 		memset(dp, 0, sizeof(*dp));
 		dp->dp_saddr = conn->c_laddr;
 		dp->dp_daddr = conn->c_faddr;
 		dp->dp_protocol_major = RDS_PROTOCOL_MAJOR(protocol_version);
 		dp->dp_protocol_minor = RDS_PROTOCOL_MINOR(protocol_version);
 		dp->dp_protocol_minor_mask = cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
 		dp->dp_ack_seq = rds_ib_piggyb_ack(ic);
 		/* Advertise flow control */
 		if (ic->i_flowctl) {
 			unsigned int credits;
 			credits = IB_GET_POST_CREDITS(atomic_read(&ic->i_credits));
 			dp->dp_credit = cpu_to_be32(credits);
 			atomic_sub(IB_SET_POST_CREDITS(credits), &ic->i_credits);
 		}
 		conn_param->private_data = dp;
 		conn_param->private_data_len = sizeof(*dp);
 	}
 }
 static void rds_ib_cq_event_handler(struct ib_event *event, void *data)
 {
 	rdsdebug("event %u data %p\n", event->event, data);
 }
 static void rds_ib_qp_event_handler(struct ib_event *event, void *data)
 {
 	struct rds_connection *conn = data;
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	rdsdebug("conn %p ic %p event %u\n", conn, ic, event->event);
 	switch (event->event) {
 	case IB_EVENT_COMM_EST:
 		rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);
 		break;
 	default:
 		rds_ib_conn_error(conn, "RDS/IB: Fatal QP Event %u "
 			"- connection %pI4->%pI4, reconnecting\n",
 			event->event, &conn->c_laddr, &conn->c_faddr);
 		break;
 	}
 }
 /*
  * This needs to be very careful to not leave IS_ERR pointers around for
  * cleanup to trip over.
  */
 static int rds_ib_setup_qp(struct rds_connection *conn)
 {
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	struct ib_device *dev = ic->i_cm_id->device;
 	struct ib_qp_init_attr attr;
 	struct rds_ib_device *rds_ibdev;
 	int ret;
 	/* rds_ib_add_one creates a rds_ib_device object per IB device,
 	 * and allocates a protection domain, memory range and FMR pool
 	 * for each.  If that fails for any reason, it will not register
 	 * the rds_ibdev at all.
 	 */
 	rds_ibdev = ib_get_client_data(dev, &rds_ib_client);
 	if (rds_ibdev == NULL) {
 		if (printk_ratelimit())
 			printk(KERN_NOTICE "RDS/IB: No client_data for device %s\n",
 					dev->name);
 		return -EOPNOTSUPP;
 	}
 	if (rds_ibdev->max_wrs < ic->i_send_ring.w_nr + 1)
 		rds_ib_ring_resize(&ic->i_send_ring, rds_ibdev->max_wrs - 1);
 	if (rds_ibdev->max_wrs < ic->i_recv_ring.w_nr + 1)
 		rds_ib_ring_resize(&ic->i_recv_ring, rds_ibdev->max_wrs - 1);
 	/* Protection domain and memory range */
 	ic->i_pd = rds_ibdev->pd;
 	ic->i_mr = rds_ibdev->mr;
 	ic->i_send_cq = ib_create_cq(dev, rds_ib_send_cq_comp_handler,
 				     rds_ib_cq_event_handler, conn,
 				     ic->i_send_ring.w_nr + 1, 0);
 	if (IS_ERR(ic->i_send_cq)) {
 		ret = PTR_ERR(ic->i_send_cq);
 		ic->i_send_cq = NULL;
 		rdsdebug("ib_create_cq send failed: %d\n", ret);
 		goto out;
 	}
 	ic->i_recv_cq = ib_create_cq(dev, rds_ib_recv_cq_comp_handler,
 				     rds_ib_cq_event_handler, conn,
 				     ic->i_recv_ring.w_nr, 0);
 	if (IS_ERR(ic->i_recv_cq)) {
 		ret = PTR_ERR(ic->i_recv_cq);
 		ic->i_recv_cq = NULL;
 		rdsdebug("ib_create_cq recv failed: %d\n", ret);
 		goto out;
 	}
 	ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
 	if (ret) {
 		rdsdebug("ib_req_notify_cq send failed: %d\n", ret);
 		goto out;
 	}
 	ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
 	if (ret) {
 		rdsdebug("ib_req_notify_cq recv failed: %d\n", ret);
 		goto out;
 	}
 	/* XXX negotiate max send/recv with remote? */
 	memset(&attr, 0, sizeof(attr));
 	attr.event_handler = rds_ib_qp_event_handler;
 	attr.qp_context = conn;
 	/* + 1 to allow for the single ack message */
 	attr.cap.max_send_wr = ic->i_send_ring.w_nr + 1;
 	attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;
 	attr.cap.max_send_sge = rds_ibdev->max_sge;
 	attr.cap.max_recv_sge = RDS_IB_RECV_SGE;
 	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 	attr.qp_type = IB_QPT_RC;
 	attr.send_cq = ic->i_send_cq;
 	attr.recv_cq = ic->i_recv_cq;
 	/*
 	 * XXX this can fail if max_*_wr is too large?  Are we supposed
 	 * to back off until we get a value that the hardware can support?
 	 */
 	ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);
 	if (ret) {
 		rdsdebug("rdma_create_qp failed: %d\n", ret);
 		goto out;
 	}
 	ic->i_send_hdrs = ib_dma_alloc_coherent(dev,
 					   ic->i_send_ring.w_nr *
 						sizeof(struct rds_header),
 					   &ic->i_send_hdrs_dma, GFP_KERNEL);
 	if (ic->i_send_hdrs == NULL) {
 		ret = -ENOMEM;
 		rdsdebug("ib_dma_alloc_coherent send failed\n");
 		goto out;
 	}
 	ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,
 					   ic->i_recv_ring.w_nr *
 						sizeof(struct rds_header),
 					   &ic->i_recv_hdrs_dma, GFP_KERNEL);
 	if (ic->i_recv_hdrs == NULL) {
 		ret = -ENOMEM;
 		rdsdebug("ib_dma_alloc_coherent recv failed\n");
 		goto out;
 	}
 	ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),
 				       &ic->i_ack_dma, GFP_KERNEL);
 	if (ic->i_ack == NULL) {
 		ret = -ENOMEM;
 		rdsdebug("ib_dma_alloc_coherent ack failed\n");
 		goto out;
 	}
 	ic->i_sends = vmalloc(ic->i_send_ring.w_nr * sizeof(struct rds_ib_send_work));
 	if (ic->i_sends == NULL) {
 		ret = -ENOMEM;
 		rdsdebug("send allocation failed\n");
 		goto out;
 	}
 	memset(ic->i_sends, 0, ic->i_send_ring.w_nr * sizeof(struct rds_ib_send_work));
 	ic->i_recvs = vmalloc(ic->i_recv_ring.w_nr * sizeof(struct rds_ib_recv_work));
 	if (ic->i_recvs == NULL) {
 		ret = -ENOMEM;
 		rdsdebug("recv allocation failed\n");
 		goto out;
 	}
 	memset(ic->i_recvs, 0, ic->i_recv_ring.w_nr * sizeof(struct rds_ib_recv_work));
 	rds_ib_recv_init_ack(ic);
 	rdsdebug("conn %p pd %p mr %p cq %p %p\n", conn, ic->i_pd, ic->i_mr,
 		 ic->i_send_cq, ic->i_recv_cq);
 out:
 	return ret;
 }
 static u32 rds_ib_protocol_compatible(struct rdma_cm_event *event)
 {
 	const struct rds_ib_connect_private *dp = event->param.conn.private_data;
 	u16 common;
 	u32 version = 0;
 	/*
 	 * rdma_cm private data is odd - when there is any private data in the
 	 * request, we will be given a pretty large buffer without telling us the
 	 * original size. The only way to tell the difference is by looking at
 	 * the contents, which are initialized to zero.
 	 * If the protocol version fields aren't set, this is a connection attempt
 	 * from an older version. This could could be 3.0 or 2.0 - we can't tell.
 	 * We really should have changed this for OFED 1.3 :-(
 	 */
 	/* Be paranoid. RDS always has privdata */
 	if (!event->param.conn.private_data_len) {
 		printk(KERN_NOTICE "RDS incoming connection has no private data, "
 			"rejecting\n");
 		return 0;
 	}
 	/* Even if len is crap *now* I still want to check it. -ASG */
 	if (event->param.conn.private_data_len < sizeof (*dp)
 	    || dp->dp_protocol_major == 0)
 		return RDS_PROTOCOL_3_0;
 	common = be16_to_cpu(dp->dp_protocol_minor_mask) & RDS_IB_SUPPORTED_PROTOCOLS;
 	if (dp->dp_protocol_major == 3 && common) {
 		version = RDS_PROTOCOL_3_0;
 		while ((common >>= 1) != 0)
 			version++;
 	} else if (printk_ratelimit()) {
 		printk(KERN_NOTICE "RDS: Connection from %pI4 using "
 			"incompatible protocol version %u.%u\n",
 			&dp->dp_saddr,
 			dp->dp_protocol_major,
 			dp->dp_protocol_minor);
 	}
 	return version;
 }
 int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
 				    struct rdma_cm_event *event)
 {
 	__be64 lguid = cm_id->route.path_rec->sgid.global.interface_id;
 	__be64 fguid = cm_id->route.path_rec->dgid.global.interface_id;
 	const struct rds_ib_connect_private *dp = event->param.conn.private_data;
 	struct rds_ib_connect_private dp_rep;
 	struct rds_connection *conn = NULL;
 	struct rds_ib_connection *ic = NULL;
 	struct rdma_conn_param conn_param;
 	u32 version;
 	int err, destroy = 1;
 	/* Check whether the remote protocol version matches ours. */
 	version = rds_ib_protocol_compatible(event);
 	if (!version)
 		goto out;
 	rdsdebug("saddr %pI4 daddr %pI4 RDSv%u.%u lguid 0x%llx fguid "
 		 "0x%llx\n", &dp->dp_saddr, &dp->dp_daddr,
 		 RDS_PROTOCOL_MAJOR(version), RDS_PROTOCOL_MINOR(version),
 		 (unsigned long long)be64_to_cpu(lguid),
 		 (unsigned long long)be64_to_cpu(fguid));
 	conn = rds_conn_create(dp->dp_daddr, dp->dp_saddr, &rds_ib_transport,
 			       GFP_KERNEL);
 	if (IS_ERR(conn)) {
 		rdsdebug("rds_conn_create failed (%ld)\n", PTR_ERR(conn));
 		conn = NULL;
 		goto out;
 	}
 	/*
 	 * The connection request may occur while the
 	 * previous connection exist, e.g. in case of failover.
 	 * But as connections may be initiated simultaneously
 	 * by both hosts, we have a random backoff mechanism -
 	 * see the comment above rds_queue_reconnect()
 	 */
 	mutex_lock(&conn->c_cm_lock);
 	if (!rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_CONNECTING)) {
 		if (rds_conn_state(conn) == RDS_CONN_UP) {
 			rdsdebug("incoming connect while connecting\n");
 			rds_conn_drop(conn);
 			rds_ib_stats_inc(s_ib_listen_closed_stale);
 		} else
 		if (rds_conn_state(conn) == RDS_CONN_CONNECTING) {
 			/* Wait and see - our connect may still be succeeding */
 			rds_ib_stats_inc(s_ib_connect_raced);
 		}
 		mutex_unlock(&conn->c_cm_lock);
 		goto out;
 	}
 	ic = conn->c_transport_data;
 	rds_ib_set_protocol(conn, version);
 	rds_ib_set_flow_control(conn, be32_to_cpu(dp->dp_credit));
 	/* If the peer gave us the last packet it saw, process this as if
 	 * we had received a regular ACK. */
 	if (dp->dp_ack_seq)
 		rds_send_drop_acked(conn, be64_to_cpu(dp->dp_ack_seq), NULL);
 	BUG_ON(cm_id->context);
 	BUG_ON(ic->i_cm_id);
 	ic->i_cm_id = cm_id;
 	cm_id->context = conn;
 	/* We got halfway through setting up the ib_connection, if we
 	 * fail now, we have to take the long route out of this mess. */
 	destroy = 0;
 	err = rds_ib_setup_qp(conn);
 	if (err) {
 		rds_ib_conn_error(conn, "rds_ib_setup_qp failed (%d)\n", err);
 		goto out;
 	}
 	rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version);
 	/* rdma_accept() calls rdma_reject() internally if it fails */
 	err = rdma_accept(cm_id, &conn_param);
 	mutex_unlock(&conn->c_cm_lock);
 	if (err) {
 		rds_ib_conn_error(conn, "rdma_accept failed (%d)\n", err);
 		goto out;
 	}
 	return 0;
 out:
 	rdma_reject(cm_id, NULL, 0);
 	return destroy;
 }
 int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id)
 {
 	struct rds_connection *conn = cm_id->context;
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	struct rdma_conn_param conn_param;
 	struct rds_ib_connect_private dp;
 	int ret;
 	/* If the peer doesn't do protocol negotiation, we must
 	 * default to RDSv3.0 */
 	rds_ib_set_protocol(conn, RDS_PROTOCOL_3_0);
 	ic->i_flowctl = rds_ib_sysctl_flow_control;	/* advertise flow control */
 	ret = rds_ib_setup_qp(conn);
 	if (ret) {
 		rds_ib_conn_error(conn, "rds_ib_setup_qp failed (%d)\n", ret);
 		goto out;
 	}
 	rds_ib_cm_fill_conn_param(conn, &conn_param, &dp, RDS_PROTOCOL_VERSION);
 	ret = rdma_connect(cm_id, &conn_param);
 	if (ret)
 		rds_ib_conn_error(conn, "rdma_connect failed (%d)\n", ret);
 out:
 	/* Beware - returning non-zero tells the rdma_cm to destroy
 	 * the cm_id. We should certainly not do it as long as we still
 	 * "own" the cm_id. */
 	if (ret) {
 		if (ic->i_cm_id == cm_id)
 			ret = 0;
 	}
 	return ret;
 }
 int rds_ib_conn_connect(struct rds_connection *conn)
 {
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	struct sockaddr_in src, dest;
 	int ret;
 	/* XXX I wonder what affect the port space has */
 	/* delegate cm event handler to rdma_transport */
 	ic->i_cm_id = rdma_create_id(rds_rdma_cm_event_handler, conn,
 				     RDMA_PS_TCP);
 	if (IS_ERR(ic->i_cm_id)) {
 		ret = PTR_ERR(ic->i_cm_id);
 		ic->i_cm_id = NULL;
 		rdsdebug("rdma_create_id() failed: %d\n", ret);
 		goto out;
 	}
 	rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);
 	src.sin_family = AF_INET;
 	src.sin_addr.s_addr = (__force u32)conn->c_laddr;
 	src.sin_port = (__force u16)htons(0);
 	dest.sin_family = AF_INET;
 	dest.sin_addr.s_addr = (__force u32)conn->c_faddr;
 	dest.sin_port = (__force u16)htons(RDS_PORT);
 	ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,
 				(struct sockaddr *)&dest,
 				RDS_RDMA_RESOLVE_TIMEOUT_MS);
 	if (ret) {
 		rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,
 			 ret);
 		rdma_destroy_id(ic->i_cm_id);
 		ic->i_cm_id = NULL;
 	}
 out:
 	return ret;
 }
 /*
  * This is so careful about only cleaning up resources that were built up
  * so that it can be called at any point during startup.  In fact it
  * can be called multiple times for a given connection.
  */
 void rds_ib_conn_shutdown(struct rds_connection *conn)
 {
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	int err = 0;
 	rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,
 		 ic->i_pd, ic->i_send_cq, ic->i_recv_cq,
 		 ic->i_cm_id ? ic->i_cm_id->qp : NULL);
 	if (ic->i_cm_id) {
 		struct ib_device *dev = ic->i_cm_id->device;
 		rdsdebug("disconnecting cm %p\n", ic->i_cm_id);
 		err = rdma_disconnect(ic->i_cm_id);
 		if (err) {
 			/* Actually this may happen quite frequently, when
 			 * an outgoing connect raced with an incoming connect.
 			 */
 			rdsdebug("failed to disconnect, cm: %p err %d\n",
 				ic->i_cm_id, err);
 		}
 		wait_event(rds_ib_ring_empty_wait,
 			rds_ib_ring_empty(&ic->i_send_ring) &&
 			rds_ib_ring_empty(&ic->i_recv_ring));
 		if (ic->i_send_hdrs)
 			ib_dma_free_coherent(dev,
 					   ic->i_send_ring.w_nr *
 						sizeof(struct rds_header),
 					   ic->i_send_hdrs,
 					   ic->i_send_hdrs_dma);
 		if (ic->i_recv_hdrs)
 			ib_dma_free_coherent(dev,
 					   ic->i_recv_ring.w_nr *
 						sizeof(struct rds_header),
 					   ic->i_recv_hdrs,
 					   ic->i_recv_hdrs_dma);
 		if (ic->i_ack)
 			ib_dma_free_coherent(dev, sizeof(struct rds_header),
 					     ic->i_ack, ic->i_ack_dma);
 		if (ic->i_sends)
 			rds_ib_send_clear_ring(ic);
 		if (ic->i_recvs)
 			rds_ib_recv_clear_ring(ic);
 		if (ic->i_cm_id->qp)
 			rdma_destroy_qp(ic->i_cm_id);
 		if (ic->i_send_cq)
 			ib_destroy_cq(ic->i_send_cq);
 		if (ic->i_recv_cq)
 			ib_destroy_cq(ic->i_recv_cq);
 		rdma_destroy_id(ic->i_cm_id);
 		/*
 		 * Move connection back to the nodev list.
 		 */
 		if (ic->rds_ibdev)
 			rds_ib_remove_conn(ic->rds_ibdev, conn);
 		ic->i_cm_id = NULL;
 		ic->i_pd = NULL;
 		ic->i_mr = NULL;
 		ic->i_send_cq = NULL;
 		ic->i_recv_cq = NULL;
 		ic->i_send_hdrs = NULL;
 		ic->i_recv_hdrs = NULL;
 		ic->i_ack = NULL;
 	}
 	BUG_ON(ic->rds_ibdev);
 	/* Clear pending transmit */
 	if (ic->i_rm) {
 		rds_message_put(ic->i_rm);
 		ic->i_rm = NULL;
 	}
 	/* Clear the ACK state */
 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
 #ifdef KERNEL_HAS_ATOMIC64
 	atomic64_set(&ic->i_ack_next, 0);
 #else
 	ic->i_ack_next = 0;
 #endif
 	ic->i_ack_recv = 0;
 	/* Clear flow control state */
 	ic->i_flowctl = 0;
 	atomic_set(&ic->i_credits, 0);
 	rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);
 	rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);
 	if (ic->i_ibinc) {
 		rds_inc_put(&ic->i_ibinc->ii_inc);
 		ic->i_ibinc = NULL;
 	}
 	vfree(ic->i_sends);
 	ic->i_sends = NULL;
 	vfree(ic->i_recvs);
 	ic->i_recvs = NULL;
 }
 int rds_ib_conn_alloc(struct rds_connection *conn, gfp_t gfp)
 {
 	struct rds_ib_connection *ic;
 	unsigned long flags;
 	/* XXX too lazy? */
 	ic = kzalloc(sizeof(struct rds_ib_connection), GFP_KERNEL);
 	if (ic == NULL)
 		return -ENOMEM;
 	INIT_LIST_HEAD(&ic->ib_node);
+	tasklet_init(&ic->i_recv_tasklet, rds_ib_recv_tasklet_fn,
+		     (unsigned long) ic);
 	mutex_init(&ic->i_recv_mutex);
 #ifndef KERNEL_HAS_ATOMIC64
 	spin_lock_init(&ic->i_ack_lock);
 #endif
 	/*
 	 * rds_ib_conn_shutdown() waits for these to be emptied so they
 	 * must be initialized before it can be called.
 	 */
 	rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);
 	rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);
 	ic->conn = conn;
 	conn->c_transport_data = ic;
 	spin_lock_irqsave(&ib_nodev_conns_lock, flags);
 	list_add_tail(&ic->ib_node, &ib_nodev_conns);
 	spin_unlock_irqrestore(&ib_nodev_conns_lock, flags);
 	rdsdebug("conn %p conn ic %p\n", conn, conn->c_transport_data);
 	return 0;
 }
 /*
  * Free a connection. Connection must be shut down and not set for reconnect.
  */
 void rds_ib_conn_free(void *arg)
 {
 	struct rds_ib_connection *ic = arg;
 	spinlock_t	*lock_ptr;
 	rdsdebug("ic %p\n", ic);
 	/*
 	 * Conn is either on a dev's list or on the nodev list.
 	 * A race with shutdown() or connect() would cause problems
 	 * (since rds_ibdev would change) but that should never happen.
 	 */
 	lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock;
 	spin_lock_irq(lock_ptr);
 	list_del(&ic->ib_node);
 	spin_unlock_irq(lock_ptr);
 	kfree(ic);
 }
 /*
  * An error occurred on the connection
  */
 void
 __rds_ib_conn_error(struct rds_connection *conn, const char *fmt, ...)
 {
 	va_list ap;
 	rds_conn_drop(conn);
 	va_start(ap, fmt);
 	vprintk(fmt, ap);
 	va_end(ap);
 }

net/rds/ib_recv.c

Diff comments View file @ d521b63

 /*
  * Copyright (c) 2006 Oracle.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
  * General Public License (GPL) Version 2, available from the file
  * COPYING in the main directory of this source tree, or the
  * OpenIB.org BSD license below:
  *
  *     Redistribution and use in source and binary forms, with or
  *     without modification, are permitted provided that the following
  *     conditions are met:
  *
  *      - Redistributions of source code must retain the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer.
  *
  *      - Redistributions in binary form must reproduce the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer in the documentation and/or other materials
  *        provided with the distribution.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  *
  */
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/dma-mapping.h>
 #include <rdma/rdma_cm.h>
 #include "rds.h"
 #include "ib.h"
 static struct kmem_cache *rds_ib_incoming_slab;
 static struct kmem_cache *rds_ib_frag_slab;
 static atomic_t	rds_ib_allocation = ATOMIC_INIT(0);
 static void rds_ib_frag_drop_page(struct rds_page_frag *frag)
 {
 	rdsdebug("frag %p page %p\n", frag, frag->f_page);
 	__free_page(frag->f_page);
 	frag->f_page = NULL;
 }
 static void rds_ib_frag_free(struct rds_page_frag *frag)
 {
 	rdsdebug("frag %p page %p\n", frag, frag->f_page);
 	BUG_ON(frag->f_page != NULL);
 	kmem_cache_free(rds_ib_frag_slab, frag);
 }
 /*
  * We map a page at a time.  Its fragments are posted in order.  This
  * is called in fragment order as the fragments get send completion events.
  * Only the last frag in the page performs the unmapping.
  *
  * It's OK for ring cleanup to call this in whatever order it likes because
  * DMA is not in flight and so we can unmap while other ring entries still
  * hold page references in their frags.
  */
 static void rds_ib_recv_unmap_page(struct rds_ib_connection *ic,
 				   struct rds_ib_recv_work *recv)
 {
 	struct rds_page_frag *frag = recv->r_frag;
 	rdsdebug("recv %p frag %p page %p\n", recv, frag, frag->f_page);
 	if (frag->f_mapped)
 		ib_dma_unmap_page(ic->i_cm_id->device,
 			       frag->f_mapped,
 			       RDS_FRAG_SIZE, DMA_FROM_DEVICE);
 	frag->f_mapped = 0;
 }
 void rds_ib_recv_init_ring(struct rds_ib_connection *ic)
 {
 	struct rds_ib_recv_work *recv;
 	u32 i;
 	for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) {
 		struct ib_sge *sge;
 		recv->r_ibinc = NULL;
 		recv->r_frag = NULL;
 		recv->r_wr.next = NULL;
 		recv->r_wr.wr_id = i;
 		recv->r_wr.sg_list = recv->r_sge;
 		recv->r_wr.num_sge = RDS_IB_RECV_SGE;
 		sge = rds_ib_data_sge(ic, recv->r_sge);
 		sge->addr = 0;
 		sge->length = RDS_FRAG_SIZE;
 		sge->lkey = ic->i_mr->lkey;
 		sge = rds_ib_header_sge(ic, recv->r_sge);
 		sge->addr = ic->i_recv_hdrs_dma + (i * sizeof(struct rds_header));
 		sge->length = sizeof(struct rds_header);
 		sge->lkey = ic->i_mr->lkey;
 	}
 }
 static void rds_ib_recv_clear_one(struct rds_ib_connection *ic,
 				  struct rds_ib_recv_work *recv)
 {
 	if (recv->r_ibinc) {
 		rds_inc_put(&recv->r_ibinc->ii_inc);
 		recv->r_ibinc = NULL;
 	}
 	if (recv->r_frag) {
 		rds_ib_recv_unmap_page(ic, recv);
 		if (recv->r_frag->f_page)
 			rds_ib_frag_drop_page(recv->r_frag);
 		rds_ib_frag_free(recv->r_frag);
 		recv->r_frag = NULL;
 	}
 }
 void rds_ib_recv_clear_ring(struct rds_ib_connection *ic)
 {
 	u32 i;
 	for (i = 0; i < ic->i_recv_ring.w_nr; i++)
 		rds_ib_recv_clear_one(ic, &ic->i_recvs[i]);
 	if (ic->i_frag.f_page)
 		rds_ib_frag_drop_page(&ic->i_frag);
 }
 static int rds_ib_recv_refill_one(struct rds_connection *conn,
 				  struct rds_ib_recv_work *recv,
 				  gfp_t kptr_gfp, gfp_t page_gfp)
 {
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	dma_addr_t dma_addr;
 	struct ib_sge *sge;
 	int ret = -ENOMEM;
 	if (recv->r_ibinc == NULL) {
 		if (!atomic_add_unless(&rds_ib_allocation, 1, rds_ib_sysctl_max_recv_allocation)) {
 			rds_ib_stats_inc(s_ib_rx_alloc_limit);
 			goto out;
 		}
 		recv->r_ibinc = kmem_cache_alloc(rds_ib_incoming_slab,
 						 kptr_gfp);
 		if (recv->r_ibinc == NULL) {
 			atomic_dec(&rds_ib_allocation);
 			goto out;
 		}
 		INIT_LIST_HEAD(&recv->r_ibinc->ii_frags);
 		rds_inc_init(&recv->r_ibinc->ii_inc, conn, conn->c_faddr);
 	}
 	if (recv->r_frag == NULL) {
 		recv->r_frag = kmem_cache_alloc(rds_ib_frag_slab, kptr_gfp);
 		if (recv->r_frag == NULL)
 			goto out;
 		INIT_LIST_HEAD(&recv->r_frag->f_item);
 		recv->r_frag->f_page = NULL;
 	}
 	if (ic->i_frag.f_page == NULL) {
 		ic->i_frag.f_page = alloc_page(page_gfp);
 		if (ic->i_frag.f_page == NULL)
 			goto out;
 		ic->i_frag.f_offset = 0;
 	}
 	dma_addr = ib_dma_map_page(ic->i_cm_id->device,
 				  ic->i_frag.f_page,
 				  ic->i_frag.f_offset,
 				  RDS_FRAG_SIZE,
 				  DMA_FROM_DEVICE);
 	if (ib_dma_mapping_error(ic->i_cm_id->device, dma_addr))
 		goto out;
 	/*
 	 * Once we get the RDS_PAGE_LAST_OFF frag then rds_ib_frag_unmap()
 	 * must be called on this recv.  This happens as completions hit
 	 * in order or on connection shutdown.
 	 */
 	recv->r_frag->f_page = ic->i_frag.f_page;
 	recv->r_frag->f_offset = ic->i_frag.f_offset;
 	recv->r_frag->f_mapped = dma_addr;
 	sge = rds_ib_data_sge(ic, recv->r_sge);
 	sge->addr = dma_addr;
 	sge->length = RDS_FRAG_SIZE;
 	sge = rds_ib_header_sge(ic, recv->r_sge);
 	sge->addr = ic->i_recv_hdrs_dma + (recv - ic->i_recvs) * sizeof(struct rds_header);
 	sge->length = sizeof(struct rds_header);
 	get_page(recv->r_frag->f_page);
 	if (ic->i_frag.f_offset < RDS_PAGE_LAST_OFF) {
 		ic->i_frag.f_offset += RDS_FRAG_SIZE;
 	} else {
 		put_page(ic->i_frag.f_page);
 		ic->i_frag.f_page = NULL;
 		ic->i_frag.f_offset = 0;
 	}
 	ret = 0;
 out:
 	return ret;
 }
 /*
  * This tries to allocate and post unused work requests after making sure that
  * they have all the allocations they need to queue received fragments into
  * sockets.  The i_recv_mutex is held here so that ring_alloc and _unalloc
  * pairs don't go unmatched.
  *
  * -1 is returned if posting fails due to temporary resource exhaustion.
  */
 int rds_ib_recv_refill(struct rds_connection *conn, gfp_t kptr_gfp,
 		       gfp_t page_gfp, int prefill)
 {
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	struct rds_ib_recv_work *recv;
 	struct ib_recv_wr *failed_wr;
 	unsigned int posted = 0;
 	int ret = 0;
 	u32 pos;
 	while ((prefill || rds_conn_up(conn))
 			&& rds_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) {
 		if (pos >= ic->i_recv_ring.w_nr) {
 			printk(KERN_NOTICE "Argh - ring alloc returned pos=%u\n",
 					pos);
 			ret = -EINVAL;
 			break;
 		}
 		recv = &ic->i_recvs[pos];
 		ret = rds_ib_recv_refill_one(conn, recv, kptr_gfp, page_gfp);
 		if (ret) {
 			ret = -1;
 			break;
 		}
 		/* XXX when can this fail? */
 		ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, &failed_wr);
 		rdsdebug("recv %p ibinc %p page %p addr %lu ret %d\n", recv,
 			 recv->r_ibinc, recv->r_frag->f_page,
 			 (long) recv->r_frag->f_mapped, ret);
 		if (ret) {
 			rds_ib_conn_error(conn, "recv post on "
 			       "%pI4 returned %d, disconnecting and "
 			       "reconnecting\n", &conn->c_faddr,
 			       ret);
 			ret = -1;
 			break;
 		}
 		posted++;
 	}
 	/* We're doing flow control - update the window. */
 	if (ic->i_flowctl && posted)
 		rds_ib_advertise_credits(conn, posted);
 	if (ret)
 		rds_ib_ring_unalloc(&ic->i_recv_ring, 1);
 	return ret;
 }
 void rds_ib_inc_purge(struct rds_incoming *inc)
 {
 	struct rds_ib_incoming *ibinc;
 	struct rds_page_frag *frag;
 	struct rds_page_frag *pos;
 	ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);
 	rdsdebug("purging ibinc %p inc %p\n", ibinc, inc);
 	list_for_each_entry_safe(frag, pos, &ibinc->ii_frags, f_item) {
 		list_del_init(&frag->f_item);
 		rds_ib_frag_drop_page(frag);
 		rds_ib_frag_free(frag);
 	}
 }
 void rds_ib_inc_free(struct rds_incoming *inc)
 {
 	struct rds_ib_incoming *ibinc;
 	ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);
 	rds_ib_inc_purge(inc);
 	rdsdebug("freeing ibinc %p inc %p\n", ibinc, inc);
 	BUG_ON(!list_empty(&ibinc->ii_frags));
 	kmem_cache_free(rds_ib_incoming_slab, ibinc);
 	atomic_dec(&rds_ib_allocation);
 	BUG_ON(atomic_read(&rds_ib_allocation) < 0);
 }
 int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iovec *first_iov,
 			    size_t size)
 {
 	struct rds_ib_incoming *ibinc;
 	struct rds_page_frag *frag;
 	struct iovec *iov = first_iov;
 	unsigned long to_copy;
 	unsigned long frag_off = 0;
 	unsigned long iov_off = 0;
 	int copied = 0;
 	int ret;
 	u32 len;
 	ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);
 	frag = list_entry(ibinc->ii_frags.next, struct rds_page_frag, f_item);
 	len = be32_to_cpu(inc->i_hdr.h_len);
 	while (copied < size && copied < len) {
 		if (frag_off == RDS_FRAG_SIZE) {
 			frag = list_entry(frag->f_item.next,
 					  struct rds_page_frag, f_item);
 			frag_off = 0;
 		}
 		while (iov_off == iov->iov_len) {
 			iov_off = 0;
 			iov++;
 		}
 		to_copy = min(iov->iov_len - iov_off, RDS_FRAG_SIZE - frag_off);
 		to_copy = min_t(size_t, to_copy, size - copied);
 		to_copy = min_t(unsigned long, to_copy, len - copied);
 		rdsdebug("%lu bytes to user [%p, %zu] + %lu from frag "
 			 "[%p, %lu] + %lu\n",
 			 to_copy, iov->iov_base, iov->iov_len, iov_off,
 			 frag->f_page, frag->f_offset, frag_off);
 		/* XXX needs + offset for multiple recvs per page */
 		ret = rds_page_copy_to_user(frag->f_page,
 					    frag->f_offset + frag_off,
 					    iov->iov_base + iov_off,
 					    to_copy);
 		if (ret) {
 			copied = ret;
 			break;
 		}
 		iov_off += to_copy;
 		frag_off += to_copy;
 		copied += to_copy;
 	}
 	return copied;
 }
 /* ic starts out kzalloc()ed */
 void rds_ib_recv_init_ack(struct rds_ib_connection *ic)
 {
 	struct ib_send_wr *wr = &ic->i_ack_wr;
 	struct ib_sge *sge = &ic->i_ack_sge;
 	sge->addr = ic->i_ack_dma;
 	sge->length = sizeof(struct rds_header);
 	sge->lkey = ic->i_mr->lkey;
 	wr->sg_list = sge;
 	wr->num_sge = 1;
 	wr->opcode = IB_WR_SEND;
 	wr->wr_id = RDS_IB_ACK_WR_ID;
 	wr->send_flags = IB_SEND_SIGNALED | IB_SEND_SOLICITED;
 }
 /*
  * You'd think that with reliable IB connections you wouldn't need to ack
  * messages that have been received.  The problem is that IB hardware generates
  * an ack message before it has DMAed the message into memory.  This creates a
  * potential message loss if the HCA is disabled for any reason between when it
  * sends the ack and before the message is DMAed and processed.  This is only a
  * potential issue if another HCA is available for fail-over.
  *
  * When the remote host receives our ack they'll free the sent message from
  * their send queue.  To decrease the latency of this we always send an ack
  * immediately after we've received messages.
  *
  * For simplicity, we only have one ack in flight at a time.  This puts
  * pressure on senders to have deep enough send queues to absorb the latency of
  * a single ack frame being in flight.  This might not be good enough.
  *
  * This is implemented by have a long-lived send_wr and sge which point to a
  * statically allocated ack frame.  This ack wr does not fall under the ring
  * accounting that the tx and rx wrs do.  The QP attribute specifically makes
  * room for it beyond the ring size.  Send completion notices its special
  * wr_id and avoids working with the ring in that case.
  */
 #ifndef KERNEL_HAS_ATOMIC64
 static void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq,
 				int ack_required)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&ic->i_ack_lock, flags);
 	ic->i_ack_next = seq;
 	if (ack_required)
 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 	spin_unlock_irqrestore(&ic->i_ack_lock, flags);
 }
 static u64 rds_ib_get_ack(struct rds_ib_connection *ic)
 {
 	unsigned long flags;
 	u64 seq;
 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 	spin_lock_irqsave(&ic->i_ack_lock, flags);
 	seq = ic->i_ack_next;
 	spin_unlock_irqrestore(&ic->i_ack_lock, flags);
 	return seq;
 }
 #else
 static void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq,
 				int ack_required)
 {
 	atomic64_set(&ic->i_ack_next, seq);
 	if (ack_required) {
 		smp_mb__before_clear_bit();
 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 	}
 }
 static u64 rds_ib_get_ack(struct rds_ib_connection *ic)
 {
 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 	smp_mb__after_clear_bit();
 	return atomic64_read(&ic->i_ack_next);
 }
 #endif
 static void rds_ib_send_ack(struct rds_ib_connection *ic, unsigned int adv_credits)
 {
 	struct rds_header *hdr = ic->i_ack;
 	struct ib_send_wr *failed_wr;
 	u64 seq;
 	int ret;
 	seq = rds_ib_get_ack(ic);
 	rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq);
 	rds_message_populate_header(hdr, 0, 0, 0);
 	hdr->h_ack = cpu_to_be64(seq);
 	hdr->h_credit = adv_credits;
 	rds_message_make_checksum(hdr);
 	ic->i_ack_queued = jiffies;
 	ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, &failed_wr);
 	if (unlikely(ret)) {
 		/* Failed to send. Release the WR, and
 		 * force another ACK.
 		 */
 		clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 		rds_ib_stats_inc(s_ib_ack_send_failure);
 		/* Need to finesse this later. */
 		BUG();
 	} else
 		rds_ib_stats_inc(s_ib_ack_sent);
 }
 /*
  * There are 3 ways of getting acknowledgements to the peer:
  *  1.	We call rds_ib_attempt_ack from the recv completion handler
  *	to send an ACK-only frame.
  *	However, there can be only one such frame in the send queue
  *	at any time, so we may have to postpone it.
  *  2.	When another (data) packet is transmitted while there's
  *	an ACK in the queue, we piggyback the ACK sequence number
  *	on the data packet.
  *  3.	If the ACK WR is done sending, we get called from the
  *	send queue completion handler, and check whether there's
  *	another ACK pending (postponed because the WR was on the
  *	queue). If so, we transmit it.
  *
  * We maintain 2 variables:
  *  -	i_ack_flags, which keeps track of whether the ACK WR
  *	is currently in the send queue or not (IB_ACK_IN_FLIGHT)
  *  -	i_ack_next, which is the last sequence number we received
  *
  * Potentially, send queue and receive queue handlers can run concurrently.
  * It would be nice to not have to use a spinlock to synchronize things,
  * but the one problem that rules this out is that 64bit updates are
  * not atomic on all platforms. Things would be a lot simpler if
  * we had atomic64 or maybe cmpxchg64 everywhere.
  *
  * Reconnecting complicates this picture just slightly. When we
  * reconnect, we may be seeing duplicate packets. The peer
  * is retransmitting them, because it hasn't seen an ACK for
  * them. It is important that we ACK these.
  *
  * ACK mitigation adds a header flag "ACK_REQUIRED"; any packet with
  * this flag set *MUST* be acknowledged immediately.
  */
 /*
  * When we get here, we're called from the recv queue handler.
  * Check whether we ought to transmit an ACK.
  */
 void rds_ib_attempt_ack(struct rds_ib_connection *ic)
 {
 	unsigned int adv_credits;
 	if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))
 		return;
 	if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) {
 		rds_ib_stats_inc(s_ib_ack_send_delayed);
 		return;
 	}
 	/* Can we get a send credit? */
 	if (!rds_ib_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) {
 		rds_ib_stats_inc(s_ib_tx_throttle);
 		clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
 		return;
 	}
 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 	rds_ib_send_ack(ic, adv_credits);
 }
 /*
  * We get here from the send completion handler, when the
  * adapter tells us the ACK frame was sent.
  */
 void rds_ib_ack_send_complete(struct rds_ib_connection *ic)
 {
 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
 	rds_ib_attempt_ack(ic);
 }
 /*
  * This is called by the regular xmit code when it wants to piggyback
  * an ACK on an outgoing frame.
  */
 u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic)
 {
 	if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))
 		rds_ib_stats_inc(s_ib_ack_send_piggybacked);
 	return rds_ib_get_ack(ic);
 }
 static struct rds_header *rds_ib_get_header(struct rds_connection *conn,
 					    struct rds_ib_recv_work *recv,
 					    u32 data_len)
 {
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	void *hdr_buff = &ic->i_recv_hdrs[recv - ic->i_recvs];
 	void *addr;
 	u32 misplaced_hdr_bytes;
 	/*
 	 * Support header at the front (RDS 3.1+) as well as header-at-end.
 	 *
 	 * Cases:
 	 * 1) header all in header buff (great!)
 	 * 2) header all in data page (copy all to header buff)
 	 * 3) header split across hdr buf + data page
 	 *    (move bit in hdr buff to end before copying other bit from data page)
 	 */
 	if (conn->c_version > RDS_PROTOCOL_3_0 || data_len == RDS_FRAG_SIZE)
 	        return hdr_buff;
 	if (data_len <= (RDS_FRAG_SIZE - sizeof(struct rds_header))) {
 		addr = kmap_atomic(recv->r_frag->f_page, KM_SOFTIRQ0);
 		memcpy(hdr_buff,
 		       addr + recv->r_frag->f_offset + data_len,
 		       sizeof(struct rds_header));
 		kunmap_atomic(addr, KM_SOFTIRQ0);
 		return hdr_buff;
 	}
 	misplaced_hdr_bytes = (sizeof(struct rds_header) - (RDS_FRAG_SIZE - data_len));
 	memmove(hdr_buff + misplaced_hdr_bytes, hdr_buff, misplaced_hdr_bytes);
 	addr = kmap_atomic(recv->r_frag->f_page, KM_SOFTIRQ0);
 	memcpy(hdr_buff, addr + recv->r_frag->f_offset + data_len,
 	       sizeof(struct rds_header) - misplaced_hdr_bytes);
 	kunmap_atomic(addr, KM_SOFTIRQ0);
 	return hdr_buff;
 }
 /*
  * It's kind of lame that we're copying from the posted receive pages into
  * long-lived bitmaps.  We could have posted the bitmaps and rdma written into
  * them.  But receiving new congestion bitmaps should be a *rare* event, so
  * hopefully we won't need to invest that complexity in making it more
  * efficient.  By copying we can share a simpler core with TCP which has to
  * copy.
  */
 static void rds_ib_cong_recv(struct rds_connection *conn,
 			      struct rds_ib_incoming *ibinc)
 {
 	struct rds_cong_map *map;
 	unsigned int map_off;
 	unsigned int map_page;
 	struct rds_page_frag *frag;
 	unsigned long frag_off;
 	unsigned long to_copy;
 	unsigned long copied;
 	uint64_t uncongested = 0;
 	void *addr;
 	/* catch completely corrupt packets */
 	if (be32_to_cpu(ibinc->ii_inc.i_hdr.h_len) != RDS_CONG_MAP_BYTES)
 		return;
 	map = conn->c_fcong;
 	map_page = 0;
 	map_off = 0;
 	frag = list_entry(ibinc->ii_frags.next, struct rds_page_frag, f_item);
 	frag_off = 0;
 	copied = 0;
 	while (copied < RDS_CONG_MAP_BYTES) {
 		uint64_t *src, *dst;
 		unsigned int k;
 		to_copy = min(RDS_FRAG_SIZE - frag_off, PAGE_SIZE - map_off);
 		BUG_ON(to_copy & 7); /* Must be 64bit aligned. */
 		addr = kmap_atomic(frag->f_page, KM_SOFTIRQ0);
 		src = addr + frag_off;
 		dst = (void *)map->m_page_addrs[map_page] + map_off;
 		for (k = 0; k < to_copy; k += 8) {
 			/* Record ports that became uncongested, ie
 			 * bits that changed from 0 to 1. */
 			uncongested |= ~(*src) & *dst;
 			*dst++ = *src++;
 		}
 		kunmap_atomic(addr, KM_SOFTIRQ0);
 		copied += to_copy;
 		map_off += to_copy;
 		if (map_off == PAGE_SIZE) {
 			map_off = 0;
 			map_page++;
 		}
 		frag_off += to_copy;
 		if (frag_off == RDS_FRAG_SIZE) {
 			frag = list_entry(frag->f_item.next,
 					  struct rds_page_frag, f_item);
 			frag_off = 0;
 		}
 	}
 	/* the congestion map is in little endian order */
 	uncongested = le64_to_cpu(uncongested);
 	rds_cong_map_updated(map, uncongested);
 }
 /*
  * Rings are posted with all the allocations they'll need to queue the
  * incoming message to the receiving socket so this can't fail.
  * All fragments start with a header, so we can make sure we're not receiving
  * garbage, and we can tell a small 8 byte fragment from an ACK frame.
  */
 struct rds_ib_ack_state {
 	u64		ack_next;
 	u64		ack_recv;
 	unsigned int	ack_required:1;
 	unsigned int	ack_next_valid:1;
 	unsigned int	ack_recv_valid:1;
 };
 static void rds_ib_process_recv(struct rds_connection *conn,
 				struct rds_ib_recv_work *recv, u32 data_len,
 				struct rds_ib_ack_state *state)
 {
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	struct rds_ib_incoming *ibinc = ic->i_ibinc;
 	struct rds_header *ihdr, *hdr;
 	/* XXX shut down the connection if port 0,0 are seen? */
 	rdsdebug("ic %p ibinc %p recv %p byte len %u\n", ic, ibinc, recv,
 		 data_len);
 	if (data_len < sizeof(struct rds_header)) {
 		rds_ib_conn_error(conn, "incoming message "
 		       "from %pI4 didn't inclue a "
 		       "header, disconnecting and "
 		       "reconnecting\n",
 		       &conn->c_faddr);
 		return;
 	}
 	data_len -= sizeof(struct rds_header);
 	ihdr = rds_ib_get_header(conn, recv, data_len);
 	/* Validate the checksum. */
 	if (!rds_message_verify_checksum(ihdr)) {
 		rds_ib_conn_error(conn, "incoming message "
 		       "from %pI4 has corrupted header - "
 		       "forcing a reconnect\n",
 		       &conn->c_faddr);
 		rds_stats_inc(s_recv_drop_bad_checksum);
 		return;
 	}
 	/* Process the ACK sequence which comes with every packet */
 	state->ack_recv = be64_to_cpu(ihdr->h_ack);
 	state->ack_recv_valid = 1;
 	/* Process the credits update if there was one */
 	if (ihdr->h_credit)
 		rds_ib_send_add_credits(conn, ihdr->h_credit);
 	if (ihdr->h_sport == 0 && ihdr->h_dport == 0 && data_len == 0) {
 		/* This is an ACK-only packet. The fact that it gets
 		 * special treatment here is that historically, ACKs
 		 * were rather special beasts.
 		 */
 		rds_ib_stats_inc(s_ib_ack_received);
 		/*
 		 * Usually the frags make their way on to incs and are then freed as
 		 * the inc is freed.  We don't go that route, so we have to drop the
 		 * page ref ourselves.  We can't just leave the page on the recv
 		 * because that confuses the dma mapping of pages and each recv's use
 		 * of a partial page.  We can leave the frag, though, it will be
 		 * reused.
 		 *
 		 * FIXME: Fold this into the code path below.
 		 */
 		rds_ib_frag_drop_page(recv->r_frag);
 		return;
 	}
 	/*
 	 * If we don't already have an inc on the connection then this
 	 * fragment has a header and starts a message.. copy its header
 	 * into the inc and save the inc so we can hang upcoming fragments
 	 * off its list.
 	 */
 	if (ibinc == NULL) {
 		ibinc = recv->r_ibinc;
 		recv->r_ibinc = NULL;
 		ic->i_ibinc = ibinc;
 		hdr = &ibinc->ii_inc.i_hdr;
 		memcpy(hdr, ihdr, sizeof(*hdr));
 		ic->i_recv_data_rem = be32_to_cpu(hdr->h_len);
 		rdsdebug("ic %p ibinc %p rem %u flag 0x%x\n", ic, ibinc,
 			 ic->i_recv_data_rem, hdr->h_flags);
 	} else {
 		hdr = &ibinc->ii_inc.i_hdr;
 		/* We can't just use memcmp here; fragments of a
 		 * single message may carry different ACKs */
 		if (hdr->h_sequence != ihdr->h_sequence
 		 || hdr->h_len != ihdr->h_len
 		 || hdr->h_sport != ihdr->h_sport
 		 || hdr->h_dport != ihdr->h_dport) {
 			rds_ib_conn_error(conn,
 				"fragment header mismatch; forcing reconnect\n");
 			return;
 		}
 	}
 	list_add_tail(&recv->r_frag->f_item, &ibinc->ii_frags);
 	recv->r_frag = NULL;
 	if (ic->i_recv_data_rem > RDS_FRAG_SIZE)
 		ic->i_recv_data_rem -= RDS_FRAG_SIZE;
 	else {
 		ic->i_recv_data_rem = 0;
 		ic->i_ibinc = NULL;
 		if (ibinc->ii_inc.i_hdr.h_flags == RDS_FLAG_CONG_BITMAP)
 			rds_ib_cong_recv(conn, ibinc);
 		else {
 			rds_recv_incoming(conn, conn->c_faddr, conn->c_laddr,
 					  &ibinc->ii_inc, GFP_ATOMIC,
 					  KM_SOFTIRQ0);
 			state->ack_next = be64_to_cpu(hdr->h_sequence);
 			state->ack_next_valid = 1;
 		}
 		/* Evaluate the ACK_REQUIRED flag *after* we received
 		 * the complete frame, and after bumping the next_rx
 		 * sequence. */
 		if (hdr->h_flags & RDS_FLAG_ACK_REQUIRED) {
 			rds_stats_inc(s_recv_ack_required);
 			state->ack_required = 1;
 		}
 		rds_inc_put(&ibinc->ii_inc);
 	}
 }
 /*
  * Plucking the oldest entry from the ring can be done concurrently with
  * the thread refilling the ring.  Each ring operation is protected by
  * spinlocks and the transient state of refilling doesn't change the
  * recording of which entry is oldest.
  *
  * This relies on IB only calling one cq comp_handler for each cq so that
  * there will only be one caller of rds_recv_incoming() per RDS connection.
  */
 void rds_ib_recv_cq_comp_handler(struct ib_cq *cq, void *context)
 {
 	struct rds_connection *conn = context;
 	struct rds_ib_connection *ic = conn->c_transport_data;
-	struct ib_wc wc;
-	struct rds_ib_ack_state state = { 0, };
-	struct rds_ib_recv_work *recv;
 	rdsdebug("conn %p cq %p\n", conn, cq);
 	rds_ib_stats_inc(s_ib_rx_cq_call);
-	ib_req_notify_cq(cq, IB_CQ_SOLICITED);
+	tasklet_schedule(&ic->i_recv_tasklet);
+}
-	while (ib_poll_cq(cq, 1, &wc) > 0) {
+static inline void rds_poll_cq(struct rds_ib_connection *ic,
+			       struct rds_ib_ack_state *state)
+{
+	struct rds_connection *conn = ic->conn;
+	struct ib_wc wc;
+	struct rds_ib_recv_work *recv;
+	while (ib_poll_cq(ic->i_recv_cq, 1, &wc) > 0) {
 		rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
 			 (unsigned long long)wc.wr_id, wc.status, wc.byte_len,
 			 be32_to_cpu(wc.ex.imm_data));
 		rds_ib_stats_inc(s_ib_rx_cq_event);
 		recv = &ic->i_recvs[rds_ib_ring_oldest(&ic->i_recv_ring)];
 		rds_ib_recv_unmap_page(ic, recv);
 		/*
 		 * Also process recvs in connecting state because it is possible
 		 * to get a recv completion _before_ the rdmacm ESTABLISHED
 		 * event is processed.
 		 */
 		if (rds_conn_up(conn) || rds_conn_connecting(conn)) {
 			/* We expect errors as the qp is drained during shutdown */
 			if (wc.status == IB_WC_SUCCESS) {
-				rds_ib_process_recv(conn, recv, wc.byte_len, &state);
+				rds_ib_process_recv(conn, recv, wc.byte_len, state);
 			} else {
 				rds_ib_conn_error(conn, "recv completion on "
 				       "%pI4 had status %u, disconnecting and "
 				       "reconnecting\n", &conn->c_faddr,
 				       wc.status);
 			}
 		}
 		rds_ib_ring_free(&ic->i_recv_ring, 1);
 	}
+}
+void rds_ib_recv_tasklet_fn(unsigned long data)
+{
+	struct rds_ib_connection *ic = (struct rds_ib_connection *) data;
+	struct rds_connection *conn = ic->conn;
+	struct rds_ib_ack_state state = { 0, };
+	rds_poll_cq(ic, &state);
+	ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
+	rds_poll_cq(ic, &state);
 	if (state.ack_next_valid)
 		rds_ib_set_ack(ic, state.ack_next, state.ack_required);
 	if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {
 		rds_send_drop_acked(conn, state.ack_recv, NULL);
 		ic->i_ack_recv = state.ack_recv;
 	}
 	if (rds_conn_up(conn))
 		rds_ib_attempt_ack(ic);
 	/* If we ever end up with a really empty receive ring, we're
 	 * in deep trouble, as the sender will definitely see RNR
 	 * timeouts. */
 	if (rds_ib_ring_empty(&ic->i_recv_ring))
 		rds_ib_stats_inc(s_ib_rx_ring_empty);
 	/*
 	 * If the ring is running low, then schedule the thread to refill.
 	 */
 	if (rds_ib_ring_low(&ic->i_recv_ring))
 		queue_delayed_work(rds_wq, &conn->c_recv_w, 0);
 }
 int rds_ib_recv(struct rds_connection *conn)
 {
 	struct rds_ib_connection *ic = conn->c_transport_data;
 	int ret = 0;
 	rdsdebug("conn %p\n", conn);
 	/*
 	 * If we get a temporary posting failure in this context then
 	 * we're really low and we want the caller to back off for a bit.
 	 */
 	mutex_lock(&ic->i_recv_mutex);
 	if (rds_ib_recv_refill(conn, GFP_KERNEL, GFP_HIGHUSER, 0))
 		ret = -ENOMEM;
 	else
 		rds_ib_stats_inc(s_ib_rx_refill_from_thread);
 	mutex_unlock(&ic->i_recv_mutex);
 	if (rds_conn_up(conn))
 		rds_ib_attempt_ack(ic);
 	return ret;
 }
 int __init rds_ib_recv_init(void)
 {
 	struct sysinfo si;
 	int ret = -ENOMEM;
 	/* Default to 30% of all available RAM for recv memory */
 	si_meminfo(&si);
 	rds_ib_sysctl_max_recv_allocation = si.totalram / 3 * PAGE_SIZE / RDS_FRAG_SIZE;
 	rds_ib_incoming_slab = kmem_cache_create("rds_ib_incoming",
 					sizeof(struct rds_ib_incoming),
 					0, 0, NULL);
 	if (rds_ib_incoming_slab == NULL)
 		goto out;
 	rds_ib_frag_slab = kmem_cache_create("rds_ib_frag",
 					sizeof(struct rds_page_frag),
 					0, 0, NULL);
 	if (rds_ib_frag_slab == NULL)
 		kmem_cache_destroy(rds_ib_incoming_slab);
 	else
 		ret = 0;
 out:
 	return ret;
 }
 void rds_ib_recv_exit(void)
 {
 	kmem_cache_destroy(rds_ib_incoming_slab);

net/rds/iw.h

Diff comments View file @ d521b63

 #ifndef _RDS_IW_H
 #define _RDS_IW_H
 #include <rdma/ib_verbs.h>
 #include <rdma/rdma_cm.h>
 #include "rds.h"
 #include "rdma_transport.h"
 #define RDS_FASTREG_SIZE		20
 #define RDS_FASTREG_POOL_SIZE		2048
 #define RDS_IW_MAX_SGE			8
 #define RDS_IW_RECV_SGE 		2
 #define RDS_IW_DEFAULT_RECV_WR		1024
 #define RDS_IW_DEFAULT_SEND_WR		256
 #define RDS_IW_SUPPORTED_PROTOCOLS	0x00000003	/* minor versions supported */
 extern struct list_head rds_iw_devices;
 /*
  * IB posts RDS_FRAG_SIZE fragments of pages to the receive queues to
  * try and minimize the amount of memory tied up both the device and
  * socket receive queues.
  */
 /* page offset of the final full frag that fits in the page */
 #define RDS_PAGE_LAST_OFF (((PAGE_SIZE  / RDS_FRAG_SIZE) - 1) * RDS_FRAG_SIZE)
 struct rds_page_frag {
 	struct list_head	f_item;
 	struct page		*f_page;
 	unsigned long		f_offset;
 	dma_addr_t 		f_mapped;
 };
 struct rds_iw_incoming {
 	struct list_head	ii_frags;
 	struct rds_incoming	ii_inc;
 };
 struct rds_iw_connect_private {
 	/* Add new fields at the end, and don't permute existing fields. */
 	__be32			dp_saddr;
 	__be32			dp_daddr;
 	u8			dp_protocol_major;
 	u8			dp_protocol_minor;
 	__be16			dp_protocol_minor_mask; /* bitmask */
 	__be32			dp_reserved1;
 	__be64			dp_ack_seq;
 	__be32			dp_credit;		/* non-zero enables flow ctl */
 };
 struct rds_iw_scatterlist {
 	struct scatterlist	*list;
 	unsigned int		len;
 	int			dma_len;
 	unsigned int		dma_npages;
 	unsigned int		bytes;
 };
 struct rds_iw_mapping {
 	spinlock_t		m_lock;	/* protect the mapping struct */
 	struct list_head	m_list;
 	struct rds_iw_mr	*m_mr;
 	uint32_t		m_rkey;
 	struct rds_iw_scatterlist m_sg;
 };
 struct rds_iw_send_work {
 	struct rds_message	*s_rm;
 	/* We should really put these into a union: */
 	struct rds_rdma_op	*s_op;
 	struct rds_iw_mapping	*s_mapping;
 	struct ib_mr		*s_mr;
 	struct ib_fast_reg_page_list *s_page_list;
 	unsigned char		s_remap_count;
 	struct ib_send_wr	s_wr;
 	struct ib_sge		s_sge[RDS_IW_MAX_SGE];
 	unsigned long		s_queued;
 };
 struct rds_iw_recv_work {
 	struct rds_iw_incoming 	*r_iwinc;
 	struct rds_page_frag	*r_frag;
 	struct ib_recv_wr	r_wr;
 	struct ib_sge		r_sge[2];
 };
 struct rds_iw_work_ring {
 	u32		w_nr;
 	u32		w_alloc_ptr;
 	u32		w_alloc_ctr;
 	u32		w_free_ptr;
 	atomic_t	w_free_ctr;
 };
 struct rds_iw_device;
 struct rds_iw_connection {
 	struct list_head	iw_node;
 	struct rds_iw_device 	*rds_iwdev;
 	struct rds_connection	*conn;
 	/* alphabet soup, IBTA style */
 	struct rdma_cm_id	*i_cm_id;
 	struct ib_pd		*i_pd;
 	struct ib_mr		*i_mr;
 	struct ib_cq		*i_send_cq;
 	struct ib_cq		*i_recv_cq;
 	/* tx */
 	struct rds_iw_work_ring	i_send_ring;
 	struct rds_message	*i_rm;
 	struct rds_header	*i_send_hdrs;
 	u64			i_send_hdrs_dma;
 	struct rds_iw_send_work *i_sends;
 	/* rx */
+	struct tasklet_struct	i_recv_tasklet;
 	struct mutex		i_recv_mutex;
 	struct rds_iw_work_ring	i_recv_ring;
 	struct rds_iw_incoming	*i_iwinc;
 	u32			i_recv_data_rem;
 	struct rds_header	*i_recv_hdrs;
 	u64			i_recv_hdrs_dma;
 	struct rds_iw_recv_work *i_recvs;
 	struct rds_page_frag	i_frag;
 	u64			i_ack_recv;	/* last ACK received */
 	/* sending acks */
 	unsigned long		i_ack_flags;
 #ifdef KERNEL_HAS_ATOMIC64
 	atomic64_t		i_ack_next;	/* next ACK to send */
 #else
 	spinlock_t		i_ack_lock;	/* protect i_ack_next */
 	u64			i_ack_next;	/* next ACK to send */
 #endif
 	struct rds_header	*i_ack;
 	struct ib_send_wr	i_ack_wr;
 	struct ib_sge		i_ack_sge;
 	u64			i_ack_dma;
 	unsigned long		i_ack_queued;
 	/* Flow control related information
 	 *
 	 * Our algorithm uses a pair variables that we need to access
 	 * atomically - one for the send credits, and one posted
 	 * recv credits we need to transfer to remote.
 	 * Rather than protect them using a slow spinlock, we put both into
 	 * a single atomic_t and update it using cmpxchg
 	 */
 	atomic_t		i_credits;
 	/* Protocol version specific information */
 	unsigned int		i_flowctl:1;	/* enable/disable flow ctl */
 	unsigned int		i_dma_local_lkey:1;
 	unsigned int		i_fastreg_posted:1; /* fastreg posted on this connection */
 	/* Batched completions */
 	unsigned int		i_unsignaled_wrs;
 	long			i_unsignaled_bytes;
 };
 /* This assumes that atomic_t is at least 32 bits */
 #define IB_GET_SEND_CREDITS(v)	((v) & 0xffff)
 #define IB_GET_POST_CREDITS(v)	((v) >> 16)
 #define IB_SET_SEND_CREDITS(v)	((v) & 0xffff)
 #define IB_SET_POST_CREDITS(v)	((v) << 16)
 struct rds_iw_cm_id {
 	struct list_head	list;
 	struct rdma_cm_id	*cm_id;
 };
 struct rds_iw_device {
 	struct list_head	list;
 	struct list_head	cm_id_list;
 	struct list_head	conn_list;
 	struct ib_device	*dev;
 	struct ib_pd		*pd;
 	struct ib_mr		*mr;
 	struct rds_iw_mr_pool	*mr_pool;
 	int			max_sge;
 	unsigned int		max_wrs;
 	unsigned int		dma_local_lkey:1;
 	spinlock_t		spinlock;	/* protect the above */
 };
 /* bits for i_ack_flags */
 #define IB_ACK_IN_FLIGHT	0
 #define IB_ACK_REQUESTED	1
 /* Magic WR_ID for ACKs */
 #define RDS_IW_ACK_WR_ID	((u64)0xffffffffffffffffULL)
 #define RDS_IW_FAST_REG_WR_ID	((u64)0xefefefefefefefefULL)
 #define RDS_IW_LOCAL_INV_WR_ID	((u64)0xdfdfdfdfdfdfdfdfULL)
 struct rds_iw_statistics {
 	uint64_t	s_iw_connect_raced;
 	uint64_t	s_iw_listen_closed_stale;
 	uint64_t	s_iw_tx_cq_call;
 	uint64_t	s_iw_tx_cq_event;
 	uint64_t	s_iw_tx_ring_full;
 	uint64_t	s_iw_tx_throttle;
 	uint64_t	s_iw_tx_sg_mapping_failure;
 	uint64_t	s_iw_tx_stalled;
 	uint64_t	s_iw_tx_credit_updates;
 	uint64_t	s_iw_rx_cq_call;
 	uint64_t	s_iw_rx_cq_event;
 	uint64_t	s_iw_rx_ring_empty;
 	uint64_t	s_iw_rx_refill_from_cq;
 	uint64_t	s_iw_rx_refill_from_thread;
 	uint64_t	s_iw_rx_alloc_limit;
 	uint64_t	s_iw_rx_credit_updates;
 	uint64_t	s_iw_ack_sent;
 	uint64_t	s_iw_ack_send_failure;
 	uint64_t	s_iw_ack_send_delayed;
 	uint64_t	s_iw_ack_send_piggybacked;
 	uint64_t	s_iw_ack_received;
 	uint64_t	s_iw_rdma_mr_alloc;
 	uint64_t	s_iw_rdma_mr_free;
 	uint64_t	s_iw_rdma_mr_used;
 	uint64_t	s_iw_rdma_mr_pool_flush;
 	uint64_t	s_iw_rdma_mr_pool_wait;
 	uint64_t	s_iw_rdma_mr_pool_depleted;
 };
 extern struct workqueue_struct *rds_iw_wq;
 /*
  * Fake ib_dma_sync_sg_for_{cpu,device} as long as ib_verbs.h
  * doesn't define it.
  */
 static inline void rds_iw_dma_sync_sg_for_cpu(struct ib_device *dev,
 		struct scatterlist *sg, unsigned int sg_dma_len, int direction)
 {
 	unsigned int i;
 	for (i = 0; i < sg_dma_len; ++i) {
 		ib_dma_sync_single_for_cpu(dev,
 				ib_sg_dma_address(dev, &sg[i]),
 				ib_sg_dma_len(dev, &sg[i]),
 				direction);
 	}
 }
 #define ib_dma_sync_sg_for_cpu	rds_iw_dma_sync_sg_for_cpu
 static inline void rds_iw_dma_sync_sg_for_device(struct ib_device *dev,
 		struct scatterlist *sg, unsigned int sg_dma_len, int direction)
 {
 	unsigned int i;
 	for (i = 0; i < sg_dma_len; ++i) {
 		ib_dma_sync_single_for_device(dev,
 				ib_sg_dma_address(dev, &sg[i]),
 				ib_sg_dma_len(dev, &sg[i]),
 				direction);
 	}
 }
 #define ib_dma_sync_sg_for_device	rds_iw_dma_sync_sg_for_device
 static inline u32 rds_iw_local_dma_lkey(struct rds_iw_connection *ic)
 {
 	return ic->i_dma_local_lkey ? ic->i_cm_id->device->local_dma_lkey : ic->i_mr->lkey;
 }
 /* ib.c */
 extern struct rds_transport rds_iw_transport;
 extern void rds_iw_add_one(struct ib_device *device);
 extern void rds_iw_remove_one(struct ib_device *device);
 extern struct ib_client rds_iw_client;
 extern unsigned int fastreg_pool_size;
 extern unsigned int fastreg_message_size;
 extern spinlock_t iw_nodev_conns_lock;
 extern struct list_head iw_nodev_conns;
 /* ib_cm.c */
 int rds_iw_conn_alloc(struct rds_connection *conn, gfp_t gfp);
 void rds_iw_conn_free(void *arg);
 int rds_iw_conn_connect(struct rds_connection *conn);
 void rds_iw_conn_shutdown(struct rds_connection *conn);
 void rds_iw_state_change(struct sock *sk);
 int __init rds_iw_listen_init(void);
 void rds_iw_listen_stop(void);
 void __rds_iw_conn_error(struct rds_connection *conn, const char *, ...);
 int rds_iw_cm_handle_connect(struct rdma_cm_id *cm_id,
 			     struct rdma_cm_event *event);
 int rds_iw_cm_initiate_connect(struct rdma_cm_id *cm_id);
 void rds_iw_cm_connect_complete(struct rds_connection *conn,
 				struct rdma_cm_event *event);
 #define rds_iw_conn_error(conn, fmt...) \
 	__rds_iw_conn_error(conn, KERN_WARNING "RDS/IW: " fmt)
 /* ib_rdma.c */
 int rds_iw_update_cm_id(struct rds_iw_device *rds_iwdev, struct rdma_cm_id *cm_id);
 void rds_iw_add_conn(struct rds_iw_device *rds_iwdev, struct rds_connection *conn);
 void rds_iw_remove_conn(struct rds_iw_device *rds_iwdev, struct rds_connection *conn);
 void __rds_iw_destroy_conns(struct list_head *list, spinlock_t *list_lock);
 static inline void rds_iw_destroy_nodev_conns(void)
 {
 	__rds_iw_destroy_conns(&iw_nodev_conns, &iw_nodev_conns_lock);
 }
 static inline void rds_iw_destroy_conns(struct rds_iw_device *rds_iwdev)
 {
 	__rds_iw_destroy_conns(&rds_iwdev->conn_list, &rds_iwdev->spinlock);
 }
 struct rds_iw_mr_pool *rds_iw_create_mr_pool(struct rds_iw_device *);
 void rds_iw_get_mr_info(struct rds_iw_device *rds_iwdev, struct rds_info_rdma_connection *iinfo);
 void rds_iw_destroy_mr_pool(struct rds_iw_mr_pool *);
 void *rds_iw_get_mr(struct scatterlist *sg, unsigned long nents,
 		    struct rds_sock *rs, u32 *key_ret);
 void rds_iw_sync_mr(void *trans_private, int dir);
 void rds_iw_free_mr(void *trans_private, int invalidate);
 void rds_iw_flush_mrs(void);
 void rds_iw_remove_cm_id(struct rds_iw_device *rds_iwdev, struct rdma_cm_id *cm_id);
 /* ib_recv.c */
 int __init rds_iw_recv_init(void);
 void rds_iw_recv_exit(void);
 int rds_iw_recv(struct rds_connection *conn);
 int rds_iw_recv_refill(struct rds_connection *conn, gfp_t kptr_gfp,
 		       gfp_t page_gfp, int prefill);
 void rds_iw_inc_purge(struct rds_incoming *inc);
 void rds_iw_inc_free(struct rds_incoming *inc);
 int rds_iw_inc_copy_to_user(struct rds_incoming *inc, struct iovec *iov,
 			     size_t size);
 void rds_iw_recv_cq_comp_handler(struct ib_cq *cq, void *context);
+void rds_iw_recv_tasklet_fn(unsigned long data);
 void rds_iw_recv_init_ring(struct rds_iw_connection *ic);
 void rds_iw_recv_clear_ring(struct rds_iw_connection *ic);
 void rds_iw_recv_init_ack(struct rds_iw_connection *ic);
 void rds_iw_attempt_ack(struct rds_iw_connection *ic);
 void rds_iw_ack_send_complete(struct rds_iw_connection *ic);
 u64 rds_iw_piggyb_ack(struct rds_iw_connection *ic);
 /* ib_ring.c */
 void rds_iw_ring_init(struct rds_iw_work_ring *ring, u32 nr);
 void rds_iw_ring_resize(struct rds_iw_work_ring *ring, u32 nr);
 u32 rds_iw_ring_alloc(struct rds_iw_work_ring *ring, u32 val, u32 *pos);
 void rds_iw_ring_free(struct rds_iw_work_ring *ring, u32 val);
 void rds_iw_ring_unalloc(struct rds_iw_work_ring *ring, u32 val);
 int rds_iw_ring_empty(struct rds_iw_work_ring *ring);
 int rds_iw_ring_low(struct rds_iw_work_ring *ring);
 u32 rds_iw_ring_oldest(struct rds_iw_work_ring *ring);
 u32 rds_iw_ring_completed(struct rds_iw_work_ring *ring, u32 wr_id, u32 oldest);
 extern wait_queue_head_t rds_iw_ring_empty_wait;
 /* ib_send.c */
 void rds_iw_xmit_complete(struct rds_connection *conn);
 int rds_iw_xmit(struct rds_connection *conn, struct rds_message *rm,
 		unsigned int hdr_off, unsigned int sg, unsigned int off);
 void rds_iw_send_cq_comp_handler(struct ib_cq *cq, void *context);
 void rds_iw_send_init_ring(struct rds_iw_connection *ic);
 void rds_iw_send_clear_ring(struct rds_iw_connection *ic);
 int rds_iw_xmit_rdma(struct rds_connection *conn, struct rds_rdma_op *op);
 void rds_iw_send_add_credits(struct rds_connection *conn, unsigned int credits);
 void rds_iw_advertise_credits(struct rds_connection *conn, unsigned int posted);
 int rds_iw_send_grab_credits(struct rds_iw_connection *ic, u32 wanted,
 			     u32 *adv_credits, int need_posted, int max_posted);
 /* ib_stats.c */
 DECLARE_PER_CPU(struct rds_iw_statistics, rds_iw_stats);
 #define rds_iw_stats_inc(member) rds_stats_inc_which(rds_iw_stats, member)
 unsigned int rds_iw_stats_info_copy(struct rds_info_iterator *iter,
 				    unsigned int avail);
 /* ib_sysctl.c */
 int __init rds_iw_sysctl_init(void);
 void rds_iw_sysctl_exit(void);
 extern unsigned long rds_iw_sysctl_max_send_wr;
 extern unsigned long rds_iw_sysctl_max_recv_wr;
 extern unsigned long rds_iw_sysctl_max_unsig_wrs;
 extern unsigned long rds_iw_sysctl_max_unsig_bytes;
 extern unsigned long rds_iw_sysctl_max_recv_allocation;
 extern unsigned int rds_iw_sysctl_flow_control;
 extern ctl_table rds_iw_sysctl_table[];
 /*
  * Helper functions for getting/setting the header and data SGEs in
  * RDS packets (not RDMA)
  */
 static inline struct ib_sge *
 rds_iw_header_sge(struct rds_iw_connection *ic, struct ib_sge *sge)
 {
 	return &sge[0];
 }
 static inline struct ib_sge *
 rds_iw_data_sge(struct rds_iw_connection *ic, struct ib_sge *sge)
 {
 	return &sge[1];
 }
 #endif

net/rds/iw_cm.c

Diff comments View file @ d521b63

 /*
  * Copyright (c) 2006 Oracle.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
  * General Public License (GPL) Version 2, available from the file
  * COPYING in the main directory of this source tree, or the
  * OpenIB.org BSD license below:
  *
  *     Redistribution and use in source and binary forms, with or
  *     without modification, are permitted provided that the following
  *     conditions are met:
  *
  *      - Redistributions of source code must retain the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer.
  *
  *      - Redistributions in binary form must reproduce the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer in the documentation and/or other materials
  *        provided with the distribution.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  *
  */
 #include <linux/kernel.h>
 #include <linux/in.h>
 #include <linux/vmalloc.h>
 #include "rds.h"
 #include "iw.h"
 /*
  * Set the selected protocol version
  */
 static void rds_iw_set_protocol(struct rds_connection *conn, unsigned int version)
 {
 	conn->c_version = version;
 }
 /*
  * Set up flow control
  */
 static void rds_iw_set_flow_control(struct rds_connection *conn, u32 credits)
 {
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	if (rds_iw_sysctl_flow_control && credits != 0) {
 		/* We're doing flow control */
 		ic->i_flowctl = 1;
 		rds_iw_send_add_credits(conn, credits);
 	} else {
 		ic->i_flowctl = 0;
 	}
 }
 /*
  * Connection established.
  * We get here for both outgoing and incoming connection.
  */
 void rds_iw_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_event *event)
 {
 	const struct rds_iw_connect_private *dp = NULL;
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	struct rds_iw_device *rds_iwdev;
 	int err;
 	if (event->param.conn.private_data_len) {
 		dp = event->param.conn.private_data;
 		rds_iw_set_protocol(conn,
 				RDS_PROTOCOL(dp->dp_protocol_major,
 					dp->dp_protocol_minor));
 		rds_iw_set_flow_control(conn, be32_to_cpu(dp->dp_credit));
 	}
 	/* update ib_device with this local ipaddr & conn */
 	rds_iwdev = ib_get_client_data(ic->i_cm_id->device, &rds_iw_client);
 	err = rds_iw_update_cm_id(rds_iwdev, ic->i_cm_id);
 	if (err)
 		printk(KERN_ERR "rds_iw_update_ipaddr failed (%d)\n", err);
 	rds_iw_add_conn(rds_iwdev, conn);
 	/* If the peer gave us the last packet it saw, process this as if
 	 * we had received a regular ACK. */
 	if (dp && dp->dp_ack_seq)
 		rds_send_drop_acked(conn, be64_to_cpu(dp->dp_ack_seq), NULL);
 	printk(KERN_NOTICE "RDS/IW: connected to %pI4<->%pI4 version %u.%u%s\n",
 			&conn->c_laddr, &conn->c_faddr,
 			RDS_PROTOCOL_MAJOR(conn->c_version),
 			RDS_PROTOCOL_MINOR(conn->c_version),
 			ic->i_flowctl ? ", flow control" : "");
 	rds_connect_complete(conn);
 }
 static void rds_iw_cm_fill_conn_param(struct rds_connection *conn,
 			struct rdma_conn_param *conn_param,
 			struct rds_iw_connect_private *dp,
 			u32 protocol_version)
 {
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	memset(conn_param, 0, sizeof(struct rdma_conn_param));
 	/* XXX tune these? */
 	conn_param->responder_resources = 1;
 	conn_param->initiator_depth = 1;
 	if (dp) {
 		memset(dp, 0, sizeof(*dp));
 		dp->dp_saddr = conn->c_laddr;
 		dp->dp_daddr = conn->c_faddr;
 		dp->dp_protocol_major = RDS_PROTOCOL_MAJOR(protocol_version);
 		dp->dp_protocol_minor = RDS_PROTOCOL_MINOR(protocol_version);
 		dp->dp_protocol_minor_mask = cpu_to_be16(RDS_IW_SUPPORTED_PROTOCOLS);
 		dp->dp_ack_seq = rds_iw_piggyb_ack(ic);
 		/* Advertise flow control */
 		if (ic->i_flowctl) {
 			unsigned int credits;
 			credits = IB_GET_POST_CREDITS(atomic_read(&ic->i_credits));
 			dp->dp_credit = cpu_to_be32(credits);
 			atomic_sub(IB_SET_POST_CREDITS(credits), &ic->i_credits);
 		}
 		conn_param->private_data = dp;
 		conn_param->private_data_len = sizeof(*dp);
 	}
 }
 static void rds_iw_cq_event_handler(struct ib_event *event, void *data)
 {
 	rdsdebug("event %u data %p\n", event->event, data);
 }
 static void rds_iw_qp_event_handler(struct ib_event *event, void *data)
 {
 	struct rds_connection *conn = data;
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	rdsdebug("conn %p ic %p event %u\n", conn, ic, event->event);
 	switch (event->event) {
 	case IB_EVENT_COMM_EST:
 		rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);
 		break;
 	case IB_EVENT_QP_REQ_ERR:
 	case IB_EVENT_QP_FATAL:
 	default:
 		rds_iw_conn_error(conn, "RDS/IW: Fatal QP Event %u - connection %pI4->%pI4...reconnecting\n",
 			event->event, &conn->c_laddr,
 			&conn->c_faddr);
 		break;
 	}
 }
 /*
  * Create a QP
  */
 static int rds_iw_init_qp_attrs(struct ib_qp_init_attr *attr,
 		struct rds_iw_device *rds_iwdev,
 		struct rds_iw_work_ring *send_ring,
 		void (*send_cq_handler)(struct ib_cq *, void *),
 		struct rds_iw_work_ring *recv_ring,
 		void (*recv_cq_handler)(struct ib_cq *, void *),
 		void *context)
 {
 	struct ib_device *dev = rds_iwdev->dev;
 	unsigned int send_size, recv_size;
 	int ret;
 	/* The offset of 1 is to accomodate the additional ACK WR. */
 	send_size = min_t(unsigned int, rds_iwdev->max_wrs, rds_iw_sysctl_max_send_wr + 1);
 	recv_size = min_t(unsigned int, rds_iwdev->max_wrs, rds_iw_sysctl_max_recv_wr + 1);
 	rds_iw_ring_resize(send_ring, send_size - 1);
 	rds_iw_ring_resize(recv_ring, recv_size - 1);
 	memset(attr, 0, sizeof(*attr));
 	attr->event_handler = rds_iw_qp_event_handler;
 	attr->qp_context = context;
 	attr->cap.max_send_wr = send_size;
 	attr->cap.max_recv_wr = recv_size;
 	attr->cap.max_send_sge = rds_iwdev->max_sge;
 	attr->cap.max_recv_sge = RDS_IW_RECV_SGE;
 	attr->sq_sig_type = IB_SIGNAL_REQ_WR;
 	attr->qp_type = IB_QPT_RC;
 	attr->send_cq = ib_create_cq(dev, send_cq_handler,
 				     rds_iw_cq_event_handler,
 				     context, send_size, 0);
 	if (IS_ERR(attr->send_cq)) {
 		ret = PTR_ERR(attr->send_cq);
 		attr->send_cq = NULL;
 		rdsdebug("ib_create_cq send failed: %d\n", ret);
 		goto out;
 	}
 	attr->recv_cq = ib_create_cq(dev, recv_cq_handler,
 				     rds_iw_cq_event_handler,
 				     context, recv_size, 0);
 	if (IS_ERR(attr->recv_cq)) {
 		ret = PTR_ERR(attr->recv_cq);
 		attr->recv_cq = NULL;
 		rdsdebug("ib_create_cq send failed: %d\n", ret);
 		goto out;
 	}
 	ret = ib_req_notify_cq(attr->send_cq, IB_CQ_NEXT_COMP);
 	if (ret) {
 		rdsdebug("ib_req_notify_cq send failed: %d\n", ret);
 		goto out;
 	}
 	ret = ib_req_notify_cq(attr->recv_cq, IB_CQ_SOLICITED);
 	if (ret) {
 		rdsdebug("ib_req_notify_cq recv failed: %d\n", ret);
 		goto out;
 	}
 out:
 	if (ret) {
 		if (attr->send_cq)
 			ib_destroy_cq(attr->send_cq);
 		if (attr->recv_cq)
 			ib_destroy_cq(attr->recv_cq);
 	}
 	return ret;
 }
 /*
  * This needs to be very careful to not leave IS_ERR pointers around for
  * cleanup to trip over.
  */
 static int rds_iw_setup_qp(struct rds_connection *conn)
 {
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	struct ib_device *dev = ic->i_cm_id->device;
 	struct ib_qp_init_attr attr;
 	struct rds_iw_device *rds_iwdev;
 	int ret;
 	/* rds_iw_add_one creates a rds_iw_device object per IB device,
 	 * and allocates a protection domain, memory range and MR pool
 	 * for each.  If that fails for any reason, it will not register
 	 * the rds_iwdev at all.
 	 */
 	rds_iwdev = ib_get_client_data(dev, &rds_iw_client);
 	if (rds_iwdev == NULL) {
 		if (printk_ratelimit())
 			printk(KERN_NOTICE "RDS/IW: No client_data for device %s\n",
 					dev->name);
 		return -EOPNOTSUPP;
 	}
 	/* Protection domain and memory range */
 	ic->i_pd = rds_iwdev->pd;
 	ic->i_mr = rds_iwdev->mr;
 	ret = rds_iw_init_qp_attrs(&attr, rds_iwdev,
 			&ic->i_send_ring, rds_iw_send_cq_comp_handler,
 			&ic->i_recv_ring, rds_iw_recv_cq_comp_handler,
 			conn);
 	if (ret < 0)
 		goto out;
 	ic->i_send_cq = attr.send_cq;
 	ic->i_recv_cq = attr.recv_cq;
 	/*
 	 * XXX this can fail if max_*_wr is too large?  Are we supposed
 	 * to back off until we get a value that the hardware can support?
 	 */
 	ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);
 	if (ret) {
 		rdsdebug("rdma_create_qp failed: %d\n", ret);
 		goto out;
 	}
 	ic->i_send_hdrs = ib_dma_alloc_coherent(dev,
 					   ic->i_send_ring.w_nr *
 						sizeof(struct rds_header),
 					   &ic->i_send_hdrs_dma, GFP_KERNEL);
 	if (ic->i_send_hdrs == NULL) {
 		ret = -ENOMEM;
 		rdsdebug("ib_dma_alloc_coherent send failed\n");
 		goto out;
 	}
 	ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,
 					   ic->i_recv_ring.w_nr *
 						sizeof(struct rds_header),
 					   &ic->i_recv_hdrs_dma, GFP_KERNEL);
 	if (ic->i_recv_hdrs == NULL) {
 		ret = -ENOMEM;
 		rdsdebug("ib_dma_alloc_coherent recv failed\n");
 		goto out;
 	}
 	ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),
 				       &ic->i_ack_dma, GFP_KERNEL);
 	if (ic->i_ack == NULL) {
 		ret = -ENOMEM;
 		rdsdebug("ib_dma_alloc_coherent ack failed\n");
 		goto out;
 	}
 	ic->i_sends = vmalloc(ic->i_send_ring.w_nr * sizeof(struct rds_iw_send_work));
 	if (ic->i_sends == NULL) {
 		ret = -ENOMEM;
 		rdsdebug("send allocation failed\n");
 		goto out;
 	}
 	rds_iw_send_init_ring(ic);
 	ic->i_recvs = vmalloc(ic->i_recv_ring.w_nr * sizeof(struct rds_iw_recv_work));
 	if (ic->i_recvs == NULL) {
 		ret = -ENOMEM;
 		rdsdebug("recv allocation failed\n");
 		goto out;
 	}
 	rds_iw_recv_init_ring(ic);
 	rds_iw_recv_init_ack(ic);
 	/* Post receive buffers - as a side effect, this will update
 	 * the posted credit count. */
 	rds_iw_recv_refill(conn, GFP_KERNEL, GFP_HIGHUSER, 1);
 	rdsdebug("conn %p pd %p mr %p cq %p %p\n", conn, ic->i_pd, ic->i_mr,
 		 ic->i_send_cq, ic->i_recv_cq);
 out:
 	return ret;
 }
 static u32 rds_iw_protocol_compatible(const struct rds_iw_connect_private *dp)
 {
 	u16 common;
 	u32 version = 0;
 	/* rdma_cm private data is odd - when there is any private data in the
 	 * request, we will be given a pretty large buffer without telling us the
 	 * original size. The only way to tell the difference is by looking at
 	 * the contents, which are initialized to zero.
 	 * If the protocol version fields aren't set, this is a connection attempt
 	 * from an older version. This could could be 3.0 or 2.0 - we can't tell.
 	 * We really should have changed this for OFED 1.3 :-( */
 	if (dp->dp_protocol_major == 0)
 		return RDS_PROTOCOL_3_0;
 	common = be16_to_cpu(dp->dp_protocol_minor_mask) & RDS_IW_SUPPORTED_PROTOCOLS;
 	if (dp->dp_protocol_major == 3 && common) {
 		version = RDS_PROTOCOL_3_0;
 		while ((common >>= 1) != 0)
 			version++;
 	} else if (printk_ratelimit()) {
 		printk(KERN_NOTICE "RDS: Connection from %pI4 using "
 			"incompatible protocol version %u.%u\n",
 			&dp->dp_saddr,
 			dp->dp_protocol_major,
 			dp->dp_protocol_minor);
 	}
 	return version;
 }
 int rds_iw_cm_handle_connect(struct rdma_cm_id *cm_id,
 				    struct rdma_cm_event *event)
 {
 	const struct rds_iw_connect_private *dp = event->param.conn.private_data;
 	struct rds_iw_connect_private dp_rep;
 	struct rds_connection *conn = NULL;
 	struct rds_iw_connection *ic = NULL;
 	struct rdma_conn_param conn_param;
 	struct rds_iw_device *rds_iwdev;
 	u32 version;
 	int err, destroy = 1;
 	/* Check whether the remote protocol version matches ours. */
 	version = rds_iw_protocol_compatible(dp);
 	if (!version)
 		goto out;
 	rdsdebug("saddr %pI4 daddr %pI4 RDSv%u.%u\n",
 		 &dp->dp_saddr, &dp->dp_daddr,
 		 RDS_PROTOCOL_MAJOR(version), RDS_PROTOCOL_MINOR(version));
 	conn = rds_conn_create(dp->dp_daddr, dp->dp_saddr, &rds_iw_transport,
 			       GFP_KERNEL);
 	if (IS_ERR(conn)) {
 		rdsdebug("rds_conn_create failed (%ld)\n", PTR_ERR(conn));
 		conn = NULL;
 		goto out;
 	}
 	/*
 	 * The connection request may occur while the
 	 * previous connection exist, e.g. in case of failover.
 	 * But as connections may be initiated simultaneously
 	 * by both hosts, we have a random backoff mechanism -
 	 * see the comment above rds_queue_reconnect()
 	 */
 	mutex_lock(&conn->c_cm_lock);
 	if (!rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_CONNECTING)) {
 		if (rds_conn_state(conn) == RDS_CONN_UP) {
 			rdsdebug("incoming connect while connecting\n");
 			rds_conn_drop(conn);
 			rds_iw_stats_inc(s_iw_listen_closed_stale);
 		} else
 		if (rds_conn_state(conn) == RDS_CONN_CONNECTING) {
 			/* Wait and see - our connect may still be succeeding */
 			rds_iw_stats_inc(s_iw_connect_raced);
 		}
 		mutex_unlock(&conn->c_cm_lock);
 		goto out;
 	}
 	ic = conn->c_transport_data;
 	rds_iw_set_protocol(conn, version);
 	rds_iw_set_flow_control(conn, be32_to_cpu(dp->dp_credit));
 	/* If the peer gave us the last packet it saw, process this as if
 	 * we had received a regular ACK. */
 	if (dp->dp_ack_seq)
 		rds_send_drop_acked(conn, be64_to_cpu(dp->dp_ack_seq), NULL);
 	BUG_ON(cm_id->context);
 	BUG_ON(ic->i_cm_id);
 	ic->i_cm_id = cm_id;
 	cm_id->context = conn;
 	rds_iwdev = ib_get_client_data(cm_id->device, &rds_iw_client);
 	ic->i_dma_local_lkey = rds_iwdev->dma_local_lkey;
 	/* We got halfway through setting up the ib_connection, if we
 	 * fail now, we have to take the long route out of this mess. */
 	destroy = 0;
 	err = rds_iw_setup_qp(conn);
 	if (err) {
 		rds_iw_conn_error(conn, "rds_iw_setup_qp failed (%d)\n", err);
 		goto out;
 	}
 	rds_iw_cm_fill_conn_param(conn, &conn_param, &dp_rep, version);
 	/* rdma_accept() calls rdma_reject() internally if it fails */
 	err = rdma_accept(cm_id, &conn_param);
 	mutex_unlock(&conn->c_cm_lock);
 	if (err) {
 		rds_iw_conn_error(conn, "rdma_accept failed (%d)\n", err);
 		goto out;
 	}
 	return 0;
 out:
 	rdma_reject(cm_id, NULL, 0);
 	return destroy;
 }
 int rds_iw_cm_initiate_connect(struct rdma_cm_id *cm_id)
 {
 	struct rds_connection *conn = cm_id->context;
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	struct rdma_conn_param conn_param;
 	struct rds_iw_connect_private dp;
 	int ret;
 	/* If the peer doesn't do protocol negotiation, we must
 	 * default to RDSv3.0 */
 	rds_iw_set_protocol(conn, RDS_PROTOCOL_3_0);
 	ic->i_flowctl = rds_iw_sysctl_flow_control;	/* advertise flow control */
 	ret = rds_iw_setup_qp(conn);
 	if (ret) {
 		rds_iw_conn_error(conn, "rds_iw_setup_qp failed (%d)\n", ret);
 		goto out;
 	}
 	rds_iw_cm_fill_conn_param(conn, &conn_param, &dp, RDS_PROTOCOL_VERSION);
 	ret = rdma_connect(cm_id, &conn_param);
 	if (ret)
 		rds_iw_conn_error(conn, "rdma_connect failed (%d)\n", ret);
 out:
 	/* Beware - returning non-zero tells the rdma_cm to destroy
 	 * the cm_id. We should certainly not do it as long as we still
 	 * "own" the cm_id. */
 	if (ret) {
 		struct rds_iw_connection *ic = conn->c_transport_data;
 		if (ic->i_cm_id == cm_id)
 			ret = 0;
 	}
 	return ret;
 }
 int rds_iw_conn_connect(struct rds_connection *conn)
 {
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	struct rds_iw_device *rds_iwdev;
 	struct sockaddr_in src, dest;
 	int ret;
 	/* XXX I wonder what affect the port space has */
 	/* delegate cm event handler to rdma_transport */
 	ic->i_cm_id = rdma_create_id(rds_rdma_cm_event_handler, conn,
 				     RDMA_PS_TCP);
 	if (IS_ERR(ic->i_cm_id)) {
 		ret = PTR_ERR(ic->i_cm_id);
 		ic->i_cm_id = NULL;
 		rdsdebug("rdma_create_id() failed: %d\n", ret);
 		goto out;
 	}
 	rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);
 	src.sin_family = AF_INET;
 	src.sin_addr.s_addr = (__force u32)conn->c_laddr;
 	src.sin_port = (__force u16)htons(0);
 	/* First, bind to the local address and device. */
 	ret = rdma_bind_addr(ic->i_cm_id, (struct sockaddr *) &src);
 	if (ret) {
 		rdsdebug("rdma_bind_addr(%pI4) failed: %d\n",
 				&conn->c_laddr, ret);
 		rdma_destroy_id(ic->i_cm_id);
 		ic->i_cm_id = NULL;
 		goto out;
 	}
 	rds_iwdev = ib_get_client_data(ic->i_cm_id->device, &rds_iw_client);
 	ic->i_dma_local_lkey = rds_iwdev->dma_local_lkey;
 	dest.sin_family = AF_INET;
 	dest.sin_addr.s_addr = (__force u32)conn->c_faddr;
 	dest.sin_port = (__force u16)htons(RDS_PORT);
 	ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,
 				(struct sockaddr *)&dest,
 				RDS_RDMA_RESOLVE_TIMEOUT_MS);
 	if (ret) {
 		rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,
 			 ret);
 		rdma_destroy_id(ic->i_cm_id);
 		ic->i_cm_id = NULL;
 	}
 out:
 	return ret;
 }
 /*
  * This is so careful about only cleaning up resources that were built up
  * so that it can be called at any point during startup.  In fact it
  * can be called multiple times for a given connection.
  */
 void rds_iw_conn_shutdown(struct rds_connection *conn)
 {
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	int err = 0;
 	struct ib_qp_attr qp_attr;
 	rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,
 		 ic->i_pd, ic->i_send_cq, ic->i_recv_cq,
 		 ic->i_cm_id ? ic->i_cm_id->qp : NULL);
 	if (ic->i_cm_id) {
 		struct ib_device *dev = ic->i_cm_id->device;
 		rdsdebug("disconnecting cm %p\n", ic->i_cm_id);
 		err = rdma_disconnect(ic->i_cm_id);
 		if (err) {
 			/* Actually this may happen quite frequently, when
 			 * an outgoing connect raced with an incoming connect.
 			 */
 			rdsdebug("rds_iw_conn_shutdown: failed to disconnect,"
 				   " cm: %p err %d\n", ic->i_cm_id, err);
 		}
 		if (ic->i_cm_id->qp) {
 			qp_attr.qp_state = IB_QPS_ERR;
 			ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);
 		}
 		wait_event(rds_iw_ring_empty_wait,
 			rds_iw_ring_empty(&ic->i_send_ring) &&
 			rds_iw_ring_empty(&ic->i_recv_ring));
 		if (ic->i_send_hdrs)
 			ib_dma_free_coherent(dev,
 					   ic->i_send_ring.w_nr *
 						sizeof(struct rds_header),
 					   ic->i_send_hdrs,
 					   ic->i_send_hdrs_dma);
 		if (ic->i_recv_hdrs)
 			ib_dma_free_coherent(dev,
 					   ic->i_recv_ring.w_nr *
 						sizeof(struct rds_header),
 					   ic->i_recv_hdrs,
 					   ic->i_recv_hdrs_dma);
 		if (ic->i_ack)
 			ib_dma_free_coherent(dev, sizeof(struct rds_header),
 					     ic->i_ack, ic->i_ack_dma);
 		if (ic->i_sends)
 			rds_iw_send_clear_ring(ic);
 		if (ic->i_recvs)
 			rds_iw_recv_clear_ring(ic);
 		if (ic->i_cm_id->qp)
 			rdma_destroy_qp(ic->i_cm_id);
 		if (ic->i_send_cq)
 			ib_destroy_cq(ic->i_send_cq);
 		if (ic->i_recv_cq)
 			ib_destroy_cq(ic->i_recv_cq);
 		/*
 		 * If associated with an rds_iw_device:
 		 * 	Move connection back to the nodev list.
 		 * 	Remove cm_id from the device cm_id list.
 		 */
 		if (ic->rds_iwdev)
 			rds_iw_remove_conn(ic->rds_iwdev, conn);
 		rdma_destroy_id(ic->i_cm_id);
 		ic->i_cm_id = NULL;
 		ic->i_pd = NULL;
 		ic->i_mr = NULL;
 		ic->i_send_cq = NULL;
 		ic->i_recv_cq = NULL;
 		ic->i_send_hdrs = NULL;
 		ic->i_recv_hdrs = NULL;
 		ic->i_ack = NULL;
 	}
 	BUG_ON(ic->rds_iwdev);
 	/* Clear pending transmit */
 	if (ic->i_rm) {
 		rds_message_put(ic->i_rm);
 		ic->i_rm = NULL;
 	}
 	/* Clear the ACK state */
 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
 #ifdef KERNEL_HAS_ATOMIC64
 	atomic64_set(&ic->i_ack_next, 0);
 #else
 	ic->i_ack_next = 0;
 #endif
 	ic->i_ack_recv = 0;
 	/* Clear flow control state */
 	ic->i_flowctl = 0;
 	atomic_set(&ic->i_credits, 0);
 	rds_iw_ring_init(&ic->i_send_ring, rds_iw_sysctl_max_send_wr);
 	rds_iw_ring_init(&ic->i_recv_ring, rds_iw_sysctl_max_recv_wr);
 	if (ic->i_iwinc) {
 		rds_inc_put(&ic->i_iwinc->ii_inc);
 		ic->i_iwinc = NULL;
 	}
 	vfree(ic->i_sends);
 	ic->i_sends = NULL;
 	vfree(ic->i_recvs);
 	ic->i_recvs = NULL;
 	rdsdebug("shutdown complete\n");
 }
 int rds_iw_conn_alloc(struct rds_connection *conn, gfp_t gfp)
 {
 	struct rds_iw_connection *ic;
 	unsigned long flags;
 	/* XXX too lazy? */
 	ic = kzalloc(sizeof(struct rds_iw_connection), GFP_KERNEL);
 	if (ic == NULL)
 		return -ENOMEM;
 	INIT_LIST_HEAD(&ic->iw_node);
+	tasklet_init(&ic->i_recv_tasklet, rds_iw_recv_tasklet_fn,
+		     (unsigned long) ic);
 	mutex_init(&ic->i_recv_mutex);
 #ifndef KERNEL_HAS_ATOMIC64
 	spin_lock_init(&ic->i_ack_lock);
 #endif
 	/*
 	 * rds_iw_conn_shutdown() waits for these to be emptied so they
 	 * must be initialized before it can be called.
 	 */
 	rds_iw_ring_init(&ic->i_send_ring, rds_iw_sysctl_max_send_wr);
 	rds_iw_ring_init(&ic->i_recv_ring, rds_iw_sysctl_max_recv_wr);
 	ic->conn = conn;
 	conn->c_transport_data = ic;
 	spin_lock_irqsave(&iw_nodev_conns_lock, flags);
 	list_add_tail(&ic->iw_node, &iw_nodev_conns);
 	spin_unlock_irqrestore(&iw_nodev_conns_lock, flags);
 	rdsdebug("conn %p conn ic %p\n", conn, conn->c_transport_data);
 	return 0;
 }
 /*
  * Free a connection. Connection must be shut down and not set for reconnect.
  */
 void rds_iw_conn_free(void *arg)
 {
 	struct rds_iw_connection *ic = arg;
 	spinlock_t	*lock_ptr;
 	rdsdebug("ic %p\n", ic);
 	/*
 	 * Conn is either on a dev's list or on the nodev list.
 	 * A race with shutdown() or connect() would cause problems
 	 * (since rds_iwdev would change) but that should never happen.
 	 */
 	lock_ptr = ic->rds_iwdev ? &ic->rds_iwdev->spinlock : &iw_nodev_conns_lock;
 	spin_lock_irq(lock_ptr);
 	list_del(&ic->iw_node);
 	spin_unlock_irq(lock_ptr);
 	kfree(ic);
 }
 /*
  * An error occurred on the connection
  */
 void
 __rds_iw_conn_error(struct rds_connection *conn, const char *fmt, ...)
 {
 	va_list ap;
 	rds_conn_drop(conn);
 	va_start(ap, fmt);
 	vprintk(fmt, ap);
 	va_end(ap);
 }

net/rds/iw_recv.c

Diff comments View file @ d521b63

 /*
  * Copyright (c) 2006 Oracle.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
  * General Public License (GPL) Version 2, available from the file
  * COPYING in the main directory of this source tree, or the
  * OpenIB.org BSD license below:
  *
  *     Redistribution and use in source and binary forms, with or
  *     without modification, are permitted provided that the following
  *     conditions are met:
  *
  *      - Redistributions of source code must retain the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer.
  *
  *      - Redistributions in binary form must reproduce the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer in the documentation and/or other materials
  *        provided with the distribution.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  *
  */
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/dma-mapping.h>
 #include <rdma/rdma_cm.h>
 #include "rds.h"
 #include "iw.h"
 static struct kmem_cache *rds_iw_incoming_slab;
 static struct kmem_cache *rds_iw_frag_slab;
 static atomic_t	rds_iw_allocation = ATOMIC_INIT(0);
 static void rds_iw_frag_drop_page(struct rds_page_frag *frag)
 {
 	rdsdebug("frag %p page %p\n", frag, frag->f_page);
 	__free_page(frag->f_page);
 	frag->f_page = NULL;
 }
 static void rds_iw_frag_free(struct rds_page_frag *frag)
 {
 	rdsdebug("frag %p page %p\n", frag, frag->f_page);
 	BUG_ON(frag->f_page != NULL);
 	kmem_cache_free(rds_iw_frag_slab, frag);
 }
 /*
  * We map a page at a time.  Its fragments are posted in order.  This
  * is called in fragment order as the fragments get send completion events.
  * Only the last frag in the page performs the unmapping.
  *
  * It's OK for ring cleanup to call this in whatever order it likes because
  * DMA is not in flight and so we can unmap while other ring entries still
  * hold page references in their frags.
  */
 static void rds_iw_recv_unmap_page(struct rds_iw_connection *ic,
 				   struct rds_iw_recv_work *recv)
 {
 	struct rds_page_frag *frag = recv->r_frag;
 	rdsdebug("recv %p frag %p page %p\n", recv, frag, frag->f_page);
 	if (frag->f_mapped)
 		ib_dma_unmap_page(ic->i_cm_id->device,
 			       frag->f_mapped,
 			       RDS_FRAG_SIZE, DMA_FROM_DEVICE);
 	frag->f_mapped = 0;
 }
 void rds_iw_recv_init_ring(struct rds_iw_connection *ic)
 {
 	struct rds_iw_recv_work *recv;
 	u32 i;
 	for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) {
 		struct ib_sge *sge;
 		recv->r_iwinc = NULL;
 		recv->r_frag = NULL;
 		recv->r_wr.next = NULL;
 		recv->r_wr.wr_id = i;
 		recv->r_wr.sg_list = recv->r_sge;
 		recv->r_wr.num_sge = RDS_IW_RECV_SGE;
 		sge = rds_iw_data_sge(ic, recv->r_sge);
 		sge->addr = 0;
 		sge->length = RDS_FRAG_SIZE;
 		sge->lkey = 0;
 		sge = rds_iw_header_sge(ic, recv->r_sge);
 		sge->addr = ic->i_recv_hdrs_dma + (i * sizeof(struct rds_header));
 		sge->length = sizeof(struct rds_header);
 		sge->lkey = 0;
 	}
 }
 static void rds_iw_recv_clear_one(struct rds_iw_connection *ic,
 				  struct rds_iw_recv_work *recv)
 {
 	if (recv->r_iwinc) {
 		rds_inc_put(&recv->r_iwinc->ii_inc);
 		recv->r_iwinc = NULL;
 	}
 	if (recv->r_frag) {
 		rds_iw_recv_unmap_page(ic, recv);
 		if (recv->r_frag->f_page)
 			rds_iw_frag_drop_page(recv->r_frag);
 		rds_iw_frag_free(recv->r_frag);
 		recv->r_frag = NULL;
 	}
 }
 void rds_iw_recv_clear_ring(struct rds_iw_connection *ic)
 {
 	u32 i;
 	for (i = 0; i < ic->i_recv_ring.w_nr; i++)
 		rds_iw_recv_clear_one(ic, &ic->i_recvs[i]);
 	if (ic->i_frag.f_page)
 		rds_iw_frag_drop_page(&ic->i_frag);
 }
 static int rds_iw_recv_refill_one(struct rds_connection *conn,
 				  struct rds_iw_recv_work *recv,
 				  gfp_t kptr_gfp, gfp_t page_gfp)
 {
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	dma_addr_t dma_addr;
 	struct ib_sge *sge;
 	int ret = -ENOMEM;
 	if (recv->r_iwinc == NULL) {
 		if (!atomic_add_unless(&rds_iw_allocation, 1, rds_iw_sysctl_max_recv_allocation)) {
 			rds_iw_stats_inc(s_iw_rx_alloc_limit);
 			goto out;
 		}
 		recv->r_iwinc = kmem_cache_alloc(rds_iw_incoming_slab,
 						 kptr_gfp);
 		if (recv->r_iwinc == NULL) {
 			atomic_dec(&rds_iw_allocation);
 			goto out;
 		}
 		INIT_LIST_HEAD(&recv->r_iwinc->ii_frags);
 		rds_inc_init(&recv->r_iwinc->ii_inc, conn, conn->c_faddr);
 	}
 	if (recv->r_frag == NULL) {
 		recv->r_frag = kmem_cache_alloc(rds_iw_frag_slab, kptr_gfp);
 		if (recv->r_frag == NULL)
 			goto out;
 		INIT_LIST_HEAD(&recv->r_frag->f_item);
 		recv->r_frag->f_page = NULL;
 	}
 	if (ic->i_frag.f_page == NULL) {
 		ic->i_frag.f_page = alloc_page(page_gfp);
 		if (ic->i_frag.f_page == NULL)
 			goto out;
 		ic->i_frag.f_offset = 0;
 	}
 	dma_addr = ib_dma_map_page(ic->i_cm_id->device,
 				  ic->i_frag.f_page,
 				  ic->i_frag.f_offset,
 				  RDS_FRAG_SIZE,
 				  DMA_FROM_DEVICE);
 	if (ib_dma_mapping_error(ic->i_cm_id->device, dma_addr))
 		goto out;
 	/*
 	 * Once we get the RDS_PAGE_LAST_OFF frag then rds_iw_frag_unmap()
 	 * must be called on this recv.  This happens as completions hit
 	 * in order or on connection shutdown.
 	 */
 	recv->r_frag->f_page = ic->i_frag.f_page;
 	recv->r_frag->f_offset = ic->i_frag.f_offset;
 	recv->r_frag->f_mapped = dma_addr;
 	sge = rds_iw_data_sge(ic, recv->r_sge);
 	sge->addr = dma_addr;
 	sge->length = RDS_FRAG_SIZE;
 	sge = rds_iw_header_sge(ic, recv->r_sge);
 	sge->addr = ic->i_recv_hdrs_dma + (recv - ic->i_recvs) * sizeof(struct rds_header);
 	sge->length = sizeof(struct rds_header);
 	get_page(recv->r_frag->f_page);
 	if (ic->i_frag.f_offset < RDS_PAGE_LAST_OFF) {
 		ic->i_frag.f_offset += RDS_FRAG_SIZE;
 	} else {
 		put_page(ic->i_frag.f_page);
 		ic->i_frag.f_page = NULL;
 		ic->i_frag.f_offset = 0;
 	}
 	ret = 0;
 out:
 	return ret;
 }
 /*
  * This tries to allocate and post unused work requests after making sure that
  * they have all the allocations they need to queue received fragments into
  * sockets.  The i_recv_mutex is held here so that ring_alloc and _unalloc
  * pairs don't go unmatched.
  *
  * -1 is returned if posting fails due to temporary resource exhaustion.
  */
 int rds_iw_recv_refill(struct rds_connection *conn, gfp_t kptr_gfp,
 		       gfp_t page_gfp, int prefill)
 {
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	struct rds_iw_recv_work *recv;
 	struct ib_recv_wr *failed_wr;
 	unsigned int posted = 0;
 	int ret = 0;
 	u32 pos;
 	while ((prefill || rds_conn_up(conn))
 			&& rds_iw_ring_alloc(&ic->i_recv_ring, 1, &pos)) {
 		if (pos >= ic->i_recv_ring.w_nr) {
 			printk(KERN_NOTICE "Argh - ring alloc returned pos=%u\n",
 					pos);
 			ret = -EINVAL;
 			break;
 		}
 		recv = &ic->i_recvs[pos];
 		ret = rds_iw_recv_refill_one(conn, recv, kptr_gfp, page_gfp);
 		if (ret) {
 			ret = -1;
 			break;
 		}
 		/* XXX when can this fail? */
 		ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, &failed_wr);
 		rdsdebug("recv %p iwinc %p page %p addr %lu ret %d\n", recv,
 			 recv->r_iwinc, recv->r_frag->f_page,
 			 (long) recv->r_frag->f_mapped, ret);
 		if (ret) {
 			rds_iw_conn_error(conn, "recv post on "
 			       "%pI4 returned %d, disconnecting and "
 			       "reconnecting\n", &conn->c_faddr,
 			       ret);
 			ret = -1;
 			break;
 		}
 		posted++;
 	}
 	/* We're doing flow control - update the window. */
 	if (ic->i_flowctl && posted)
 		rds_iw_advertise_credits(conn, posted);
 	if (ret)
 		rds_iw_ring_unalloc(&ic->i_recv_ring, 1);
 	return ret;
 }
 void rds_iw_inc_purge(struct rds_incoming *inc)
 {
 	struct rds_iw_incoming *iwinc;
 	struct rds_page_frag *frag;
 	struct rds_page_frag *pos;
 	iwinc = container_of(inc, struct rds_iw_incoming, ii_inc);
 	rdsdebug("purging iwinc %p inc %p\n", iwinc, inc);
 	list_for_each_entry_safe(frag, pos, &iwinc->ii_frags, f_item) {
 		list_del_init(&frag->f_item);
 		rds_iw_frag_drop_page(frag);
 		rds_iw_frag_free(frag);
 	}
 }
 void rds_iw_inc_free(struct rds_incoming *inc)
 {
 	struct rds_iw_incoming *iwinc;
 	iwinc = container_of(inc, struct rds_iw_incoming, ii_inc);
 	rds_iw_inc_purge(inc);
 	rdsdebug("freeing iwinc %p inc %p\n", iwinc, inc);
 	BUG_ON(!list_empty(&iwinc->ii_frags));
 	kmem_cache_free(rds_iw_incoming_slab, iwinc);
 	atomic_dec(&rds_iw_allocation);
 	BUG_ON(atomic_read(&rds_iw_allocation) < 0);
 }
 int rds_iw_inc_copy_to_user(struct rds_incoming *inc, struct iovec *first_iov,
 			    size_t size)
 {
 	struct rds_iw_incoming *iwinc;
 	struct rds_page_frag *frag;
 	struct iovec *iov = first_iov;
 	unsigned long to_copy;
 	unsigned long frag_off = 0;
 	unsigned long iov_off = 0;
 	int copied = 0;
 	int ret;
 	u32 len;
 	iwinc = container_of(inc, struct rds_iw_incoming, ii_inc);
 	frag = list_entry(iwinc->ii_frags.next, struct rds_page_frag, f_item);
 	len = be32_to_cpu(inc->i_hdr.h_len);
 	while (copied < size && copied < len) {
 		if (frag_off == RDS_FRAG_SIZE) {
 			frag = list_entry(frag->f_item.next,
 					  struct rds_page_frag, f_item);
 			frag_off = 0;
 		}
 		while (iov_off == iov->iov_len) {
 			iov_off = 0;
 			iov++;
 		}
 		to_copy = min(iov->iov_len - iov_off, RDS_FRAG_SIZE - frag_off);
 		to_copy = min_t(size_t, to_copy, size - copied);
 		to_copy = min_t(unsigned long, to_copy, len - copied);
 		rdsdebug("%lu bytes to user [%p, %zu] + %lu from frag "
 			 "[%p, %lu] + %lu\n",
 			 to_copy, iov->iov_base, iov->iov_len, iov_off,
 			 frag->f_page, frag->f_offset, frag_off);
 		/* XXX needs + offset for multiple recvs per page */
 		ret = rds_page_copy_to_user(frag->f_page,
 					    frag->f_offset + frag_off,
 					    iov->iov_base + iov_off,
 					    to_copy);
 		if (ret) {
 			copied = ret;
 			break;
 		}
 		iov_off += to_copy;
 		frag_off += to_copy;
 		copied += to_copy;
 	}
 	return copied;
 }
 /* ic starts out kzalloc()ed */
 void rds_iw_recv_init_ack(struct rds_iw_connection *ic)
 {
 	struct ib_send_wr *wr = &ic->i_ack_wr;
 	struct ib_sge *sge = &ic->i_ack_sge;
 	sge->addr = ic->i_ack_dma;
 	sge->length = sizeof(struct rds_header);
 	sge->lkey = rds_iw_local_dma_lkey(ic);
 	wr->sg_list = sge;
 	wr->num_sge = 1;
 	wr->opcode = IB_WR_SEND;
 	wr->wr_id = RDS_IW_ACK_WR_ID;
 	wr->send_flags = IB_SEND_SIGNALED | IB_SEND_SOLICITED;
 }
 /*
  * You'd think that with reliable IB connections you wouldn't need to ack
  * messages that have been received.  The problem is that IB hardware generates
  * an ack message before it has DMAed the message into memory.  This creates a
  * potential message loss if the HCA is disabled for any reason between when it
  * sends the ack and before the message is DMAed and processed.  This is only a
  * potential issue if another HCA is available for fail-over.
  *
  * When the remote host receives our ack they'll free the sent message from
  * their send queue.  To decrease the latency of this we always send an ack
  * immediately after we've received messages.
  *
  * For simplicity, we only have one ack in flight at a time.  This puts
  * pressure on senders to have deep enough send queues to absorb the latency of
  * a single ack frame being in flight.  This might not be good enough.
  *
  * This is implemented by have a long-lived send_wr and sge which point to a
  * statically allocated ack frame.  This ack wr does not fall under the ring
  * accounting that the tx and rx wrs do.  The QP attribute specifically makes
  * room for it beyond the ring size.  Send completion notices its special
  * wr_id and avoids working with the ring in that case.
  */
 #ifndef KERNEL_HAS_ATOMIC64
 static void rds_iw_set_ack(struct rds_iw_connection *ic, u64 seq,
 				int ack_required)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&ic->i_ack_lock, flags);
 	ic->i_ack_next = seq;
 	if (ack_required)
 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 	spin_unlock_irqrestore(&ic->i_ack_lock, flags);
 }
 static u64 rds_iw_get_ack(struct rds_iw_connection *ic)
 {
 	unsigned long flags;
 	u64 seq;
 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 	spin_lock_irqsave(&ic->i_ack_lock, flags);
 	seq = ic->i_ack_next;
 	spin_unlock_irqrestore(&ic->i_ack_lock, flags);
 	return seq;
 }
 #else
 static void rds_iw_set_ack(struct rds_iw_connection *ic, u64 seq,
 				int ack_required)
 {
 	atomic64_set(&ic->i_ack_next, seq);
 	if (ack_required) {
 		smp_mb__before_clear_bit();
 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 	}
 }
 static u64 rds_iw_get_ack(struct rds_iw_connection *ic)
 {
 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 	smp_mb__after_clear_bit();
 	return atomic64_read(&ic->i_ack_next);
 }
 #endif
 static void rds_iw_send_ack(struct rds_iw_connection *ic, unsigned int adv_credits)
 {
 	struct rds_header *hdr = ic->i_ack;
 	struct ib_send_wr *failed_wr;
 	u64 seq;
 	int ret;
 	seq = rds_iw_get_ack(ic);
 	rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq);
 	rds_message_populate_header(hdr, 0, 0, 0);
 	hdr->h_ack = cpu_to_be64(seq);
 	hdr->h_credit = adv_credits;
 	rds_message_make_checksum(hdr);
 	ic->i_ack_queued = jiffies;
 	ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, &failed_wr);
 	if (unlikely(ret)) {
 		/* Failed to send. Release the WR, and
 		 * force another ACK.
 		 */
 		clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 		rds_iw_stats_inc(s_iw_ack_send_failure);
 		/* Need to finesse this later. */
 		BUG();
 	} else
 		rds_iw_stats_inc(s_iw_ack_sent);
 }
 /*
  * There are 3 ways of getting acknowledgements to the peer:
  *  1.	We call rds_iw_attempt_ack from the recv completion handler
  *	to send an ACK-only frame.
  *	However, there can be only one such frame in the send queue
  *	at any time, so we may have to postpone it.
  *  2.	When another (data) packet is transmitted while there's
  *	an ACK in the queue, we piggyback the ACK sequence number
  *	on the data packet.
  *  3.	If the ACK WR is done sending, we get called from the
  *	send queue completion handler, and check whether there's
  *	another ACK pending (postponed because the WR was on the
  *	queue). If so, we transmit it.
  *
  * We maintain 2 variables:
  *  -	i_ack_flags, which keeps track of whether the ACK WR
  *	is currently in the send queue or not (IB_ACK_IN_FLIGHT)
  *  -	i_ack_next, which is the last sequence number we received
  *
  * Potentially, send queue and receive queue handlers can run concurrently.
  * It would be nice to not have to use a spinlock to synchronize things,
  * but the one problem that rules this out is that 64bit updates are
  * not atomic on all platforms. Things would be a lot simpler if
  * we had atomic64 or maybe cmpxchg64 everywhere.
  *
  * Reconnecting complicates this picture just slightly. When we
  * reconnect, we may be seeing duplicate packets. The peer
  * is retransmitting them, because it hasn't seen an ACK for
  * them. It is important that we ACK these.
  *
  * ACK mitigation adds a header flag "ACK_REQUIRED"; any packet with
  * this flag set *MUST* be acknowledged immediately.
  */
 /*
  * When we get here, we're called from the recv queue handler.
  * Check whether we ought to transmit an ACK.
  */
 void rds_iw_attempt_ack(struct rds_iw_connection *ic)
 {
 	unsigned int adv_credits;
 	if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))
 		return;
 	if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) {
 		rds_iw_stats_inc(s_iw_ack_send_delayed);
 		return;
 	}
 	/* Can we get a send credit? */
 	if (!rds_iw_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) {
 		rds_iw_stats_inc(s_iw_tx_throttle);
 		clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
 		return;
 	}
 	clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
 	rds_iw_send_ack(ic, adv_credits);
 }
 /*
  * We get here from the send completion handler, when the
  * adapter tells us the ACK frame was sent.
  */
 void rds_iw_ack_send_complete(struct rds_iw_connection *ic)
 {
 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
 	rds_iw_attempt_ack(ic);
 }
 /*
  * This is called by the regular xmit code when it wants to piggyback
  * an ACK on an outgoing frame.
  */
 u64 rds_iw_piggyb_ack(struct rds_iw_connection *ic)
 {
 	if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))
 		rds_iw_stats_inc(s_iw_ack_send_piggybacked);
 	return rds_iw_get_ack(ic);
 }
 /*
  * It's kind of lame that we're copying from the posted receive pages into
  * long-lived bitmaps.  We could have posted the bitmaps and rdma written into
  * them.  But receiving new congestion bitmaps should be a *rare* event, so
  * hopefully we won't need to invest that complexity in making it more
  * efficient.  By copying we can share a simpler core with TCP which has to
  * copy.
  */
 static void rds_iw_cong_recv(struct rds_connection *conn,
 			      struct rds_iw_incoming *iwinc)
 {
 	struct rds_cong_map *map;
 	unsigned int map_off;
 	unsigned int map_page;
 	struct rds_page_frag *frag;
 	unsigned long frag_off;
 	unsigned long to_copy;
 	unsigned long copied;
 	uint64_t uncongested = 0;
 	void *addr;
 	/* catch completely corrupt packets */
 	if (be32_to_cpu(iwinc->ii_inc.i_hdr.h_len) != RDS_CONG_MAP_BYTES)
 		return;
 	map = conn->c_fcong;
 	map_page = 0;
 	map_off = 0;
 	frag = list_entry(iwinc->ii_frags.next, struct rds_page_frag, f_item);
 	frag_off = 0;
 	copied = 0;
 	while (copied < RDS_CONG_MAP_BYTES) {
 		uint64_t *src, *dst;
 		unsigned int k;
 		to_copy = min(RDS_FRAG_SIZE - frag_off, PAGE_SIZE - map_off);
 		BUG_ON(to_copy & 7); /* Must be 64bit aligned. */
 		addr = kmap_atomic(frag->f_page, KM_SOFTIRQ0);
 		src = addr + frag_off;
 		dst = (void *)map->m_page_addrs[map_page] + map_off;
 		for (k = 0; k < to_copy; k += 8) {
 			/* Record ports that became uncongested, ie
 			 * bits that changed from 0 to 1. */
 			uncongested |= ~(*src) & *dst;
 			*dst++ = *src++;
 		}
 		kunmap_atomic(addr, KM_SOFTIRQ0);
 		copied += to_copy;
 		map_off += to_copy;
 		if (map_off == PAGE_SIZE) {
 			map_off = 0;
 			map_page++;
 		}
 		frag_off += to_copy;
 		if (frag_off == RDS_FRAG_SIZE) {
 			frag = list_entry(frag->f_item.next,
 					  struct rds_page_frag, f_item);
 			frag_off = 0;
 		}
 	}
 	/* the congestion map is in little endian order */
 	uncongested = le64_to_cpu(uncongested);
 	rds_cong_map_updated(map, uncongested);
 }
 /*
  * Rings are posted with all the allocations they'll need to queue the
  * incoming message to the receiving socket so this can't fail.
  * All fragments start with a header, so we can make sure we're not receiving
  * garbage, and we can tell a small 8 byte fragment from an ACK frame.
  */
 struct rds_iw_ack_state {
 	u64		ack_next;
 	u64		ack_recv;
 	unsigned int	ack_required:1;
 	unsigned int	ack_next_valid:1;
 	unsigned int	ack_recv_valid:1;
 };
 static void rds_iw_process_recv(struct rds_connection *conn,
 				struct rds_iw_recv_work *recv, u32 byte_len,
 				struct rds_iw_ack_state *state)
 {
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	struct rds_iw_incoming *iwinc = ic->i_iwinc;
 	struct rds_header *ihdr, *hdr;
 	/* XXX shut down the connection if port 0,0 are seen? */
 	rdsdebug("ic %p iwinc %p recv %p byte len %u\n", ic, iwinc, recv,
 		 byte_len);
 	if (byte_len < sizeof(struct rds_header)) {
 		rds_iw_conn_error(conn, "incoming message "
 		       "from %pI4 didn't inclue a "
 		       "header, disconnecting and "
 		       "reconnecting\n",
 		       &conn->c_faddr);
 		return;
 	}
 	byte_len -= sizeof(struct rds_header);
 	ihdr = &ic->i_recv_hdrs[recv - ic->i_recvs];
 	/* Validate the checksum. */
 	if (!rds_message_verify_checksum(ihdr)) {
 		rds_iw_conn_error(conn, "incoming message "
 		       "from %pI4 has corrupted header - "
 		       "forcing a reconnect\n",
 		       &conn->c_faddr);
 		rds_stats_inc(s_recv_drop_bad_checksum);
 		return;
 	}
 	/* Process the ACK sequence which comes with every packet */
 	state->ack_recv = be64_to_cpu(ihdr->h_ack);
 	state->ack_recv_valid = 1;
 	/* Process the credits update if there was one */
 	if (ihdr->h_credit)
 		rds_iw_send_add_credits(conn, ihdr->h_credit);
 	if (ihdr->h_sport == 0 && ihdr->h_dport == 0 && byte_len == 0) {
 		/* This is an ACK-only packet. The fact that it gets
 		 * special treatment here is that historically, ACKs
 		 * were rather special beasts.
 		 */
 		rds_iw_stats_inc(s_iw_ack_received);
 		/*
 		 * Usually the frags make their way on to incs and are then freed as
 		 * the inc is freed.  We don't go that route, so we have to drop the
 		 * page ref ourselves.  We can't just leave the page on the recv
 		 * because that confuses the dma mapping of pages and each recv's use
 		 * of a partial page.  We can leave the frag, though, it will be
 		 * reused.
 		 *
 		 * FIXME: Fold this into the code path below.
 		 */
 		rds_iw_frag_drop_page(recv->r_frag);
 		return;
 	}
 	/*
 	 * If we don't already have an inc on the connection then this
 	 * fragment has a header and starts a message.. copy its header
 	 * into the inc and save the inc so we can hang upcoming fragments
 	 * off its list.
 	 */
 	if (iwinc == NULL) {
 		iwinc = recv->r_iwinc;
 		recv->r_iwinc = NULL;
 		ic->i_iwinc = iwinc;
 		hdr = &iwinc->ii_inc.i_hdr;
 		memcpy(hdr, ihdr, sizeof(*hdr));
 		ic->i_recv_data_rem = be32_to_cpu(hdr->h_len);
 		rdsdebug("ic %p iwinc %p rem %u flag 0x%x\n", ic, iwinc,
 			 ic->i_recv_data_rem, hdr->h_flags);
 	} else {
 		hdr = &iwinc->ii_inc.i_hdr;
 		/* We can't just use memcmp here; fragments of a
 		 * single message may carry different ACKs */
 		if (hdr->h_sequence != ihdr->h_sequence
 		 || hdr->h_len != ihdr->h_len
 		 || hdr->h_sport != ihdr->h_sport
 		 || hdr->h_dport != ihdr->h_dport) {
 			rds_iw_conn_error(conn,
 				"fragment header mismatch; forcing reconnect\n");
 			return;
 		}
 	}
 	list_add_tail(&recv->r_frag->f_item, &iwinc->ii_frags);
 	recv->r_frag = NULL;
 	if (ic->i_recv_data_rem > RDS_FRAG_SIZE)
 		ic->i_recv_data_rem -= RDS_FRAG_SIZE;
 	else {
 		ic->i_recv_data_rem = 0;
 		ic->i_iwinc = NULL;
 		if (iwinc->ii_inc.i_hdr.h_flags == RDS_FLAG_CONG_BITMAP)
 			rds_iw_cong_recv(conn, iwinc);
 		else {
 			rds_recv_incoming(conn, conn->c_faddr, conn->c_laddr,
 					  &iwinc->ii_inc, GFP_ATOMIC,
 					  KM_SOFTIRQ0);
 			state->ack_next = be64_to_cpu(hdr->h_sequence);
 			state->ack_next_valid = 1;
 		}
 		/* Evaluate the ACK_REQUIRED flag *after* we received
 		 * the complete frame, and after bumping the next_rx
 		 * sequence. */
 		if (hdr->h_flags & RDS_FLAG_ACK_REQUIRED) {
 			rds_stats_inc(s_recv_ack_required);
 			state->ack_required = 1;
 		}
 		rds_inc_put(&iwinc->ii_inc);
 	}
 }
 /*
  * Plucking the oldest entry from the ring can be done concurrently with
  * the thread refilling the ring.  Each ring operation is protected by
  * spinlocks and the transient state of refilling doesn't change the
  * recording of which entry is oldest.
  *
  * This relies on IB only calling one cq comp_handler for each cq so that
  * there will only be one caller of rds_recv_incoming() per RDS connection.
  */
 void rds_iw_recv_cq_comp_handler(struct ib_cq *cq, void *context)
 {
 	struct rds_connection *conn = context;
 	struct rds_iw_connection *ic = conn->c_transport_data;
-	struct ib_wc wc;
-	struct rds_iw_ack_state state = { 0, };
-	struct rds_iw_recv_work *recv;
 	rdsdebug("conn %p cq %p\n", conn, cq);
 	rds_iw_stats_inc(s_iw_rx_cq_call);
-	ib_req_notify_cq(cq, IB_CQ_SOLICITED);
+	tasklet_schedule(&ic->i_recv_tasklet);
+}
-	while (ib_poll_cq(cq, 1, &wc) > 0) {
+static inline void rds_poll_cq(struct rds_iw_connection *ic,
+			       struct rds_iw_ack_state *state)
+{
+	struct rds_connection *conn = ic->conn;
+	struct ib_wc wc;
+	struct rds_iw_recv_work *recv;
+	while (ib_poll_cq(ic->i_recv_cq, 1, &wc) > 0) {
 		rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
 			 (unsigned long long)wc.wr_id, wc.status, wc.byte_len,
 			 be32_to_cpu(wc.ex.imm_data));
 		rds_iw_stats_inc(s_iw_rx_cq_event);
 		recv = &ic->i_recvs[rds_iw_ring_oldest(&ic->i_recv_ring)];
 		rds_iw_recv_unmap_page(ic, recv);
 		/*
 		 * Also process recvs in connecting state because it is possible
 		 * to get a recv completion _before_ the rdmacm ESTABLISHED
 		 * event is processed.
 		 */
 		if (rds_conn_up(conn) || rds_conn_connecting(conn)) {
 			/* We expect errors as the qp is drained during shutdown */
 			if (wc.status == IB_WC_SUCCESS) {
-				rds_iw_process_recv(conn, recv, wc.byte_len, &state);
+				rds_iw_process_recv(conn, recv, wc.byte_len, state);
 			} else {
 				rds_iw_conn_error(conn, "recv completion on "
 				       "%pI4 had status %u, disconnecting and "
 				       "reconnecting\n", &conn->c_faddr,
 				       wc.status);
 			}
 		}
 		rds_iw_ring_free(&ic->i_recv_ring, 1);
 	}
+}
+void rds_iw_recv_tasklet_fn(unsigned long data)
+{
+	struct rds_iw_connection *ic = (struct rds_iw_connection *) data;
+	struct rds_connection *conn = ic->conn;
+	struct rds_iw_ack_state state = { 0, };
+	rds_poll_cq(ic, &state);
+	ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
+	rds_poll_cq(ic, &state);
 	if (state.ack_next_valid)
 		rds_iw_set_ack(ic, state.ack_next, state.ack_required);
 	if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {
 		rds_send_drop_acked(conn, state.ack_recv, NULL);
 		ic->i_ack_recv = state.ack_recv;
 	}
 	if (rds_conn_up(conn))
 		rds_iw_attempt_ack(ic);
 	/* If we ever end up with a really empty receive ring, we're
 	 * in deep trouble, as the sender will definitely see RNR
 	 * timeouts. */
 	if (rds_iw_ring_empty(&ic->i_recv_ring))
 		rds_iw_stats_inc(s_iw_rx_ring_empty);
 	/*
 	 * If the ring is running low, then schedule the thread to refill.
 	 */
 	if (rds_iw_ring_low(&ic->i_recv_ring))
 		queue_delayed_work(rds_wq, &conn->c_recv_w, 0);
 }
 int rds_iw_recv(struct rds_connection *conn)
 {
 	struct rds_iw_connection *ic = conn->c_transport_data;
 	int ret = 0;
 	rdsdebug("conn %p\n", conn);
 	/*
 	 * If we get a temporary posting failure in this context then
 	 * we're really low and we want the caller to back off for a bit.
 	 */
 	mutex_lock(&ic->i_recv_mutex);
 	if (rds_iw_recv_refill(conn, GFP_KERNEL, GFP_HIGHUSER, 0))
 		ret = -ENOMEM;
 	else
 		rds_iw_stats_inc(s_iw_rx_refill_from_thread);
 	mutex_unlock(&ic->i_recv_mutex);
 	if (rds_conn_up(conn))
 		rds_iw_attempt_ack(ic);
 	return ret;
 }
 int __init rds_iw_recv_init(void)
 {
 	struct sysinfo si;
 	int ret = -ENOMEM;
 	/* Default to 30% of all available RAM for recv memory */
 	si_meminfo(&si);
 	rds_iw_sysctl_max_recv_allocation = si.totalram / 3 * PAGE_SIZE / RDS_FRAG_SIZE;
 	rds_iw_incoming_slab = kmem_cache_create("rds_iw_incoming",
 					sizeof(struct rds_iw_incoming),
 					0, 0, NULL);
 	if (rds_iw_incoming_slab == NULL)
 		goto out;
 	rds_iw_frag_slab = kmem_cache_create("rds_iw_frag",
 					sizeof(struct rds_page_frag),
 					0, 0, NULL);
 	if (rds_iw_frag_slab == NULL)
 		kmem_cache_destroy(rds_iw_incoming_slab);
 	else
 		ret = 0;
 out:
 	return ret;
 }
 void rds_iw_recv_exit(void)
 {
 	kmem_cache_destroy(rds_iw_incoming_slab);