Commit 518b1646 authored by Michael S. Tsirkin's avatar Michael S. Tsirkin Committed by Roland Dreier

IPoIB/cm: Fix SRQ WR leak

SRQ WR leakage has been observed with IPoIB/CM: e.g. flipping ports on
and off will, with time, leak out all WRs and then all connections
will start getting RNR NAKs.  Fix this in the way suggested by spec:
move the QP being destroyed to the error state, wait for "Last WQE
Reached" event and then post WR on a "drain QP" connected to the same
CQ.  Once we observe a completion on the drain QP, it's safe to call
ib_destroy_qp.
Signed-off-by: default avatarMichael S. Tsirkin <mst@dev.mellanox.co.il>
Signed-off-by: default avatarRoland Dreier <rolandd@cisco.com>
parent 24bd1e4e
...@@ -132,12 +132,46 @@ struct ipoib_cm_data { ...@@ -132,12 +132,46 @@ struct ipoib_cm_data {
__be32 mtu; __be32 mtu;
}; };
/*
* Quoting 10.3.1 Queue Pair and EE Context States:
*
* Note, for QPs that are associated with an SRQ, the Consumer should take the
* QP through the Error State before invoking a Destroy QP or a Modify QP to the
* Reset State. The Consumer may invoke the Destroy QP without first performing
* a Modify QP to the Error State and waiting for the Affiliated Asynchronous
* Last WQE Reached Event. However, if the Consumer does not wait for the
* Affiliated Asynchronous Last WQE Reached Event, then WQE and Data Segment
* leakage may occur. Therefore, it is good programming practice to tear down a
* QP that is associated with an SRQ by using the following process:
*
* - Put the QP in the Error State
* - Wait for the Affiliated Asynchronous Last WQE Reached Event;
* - either:
* drain the CQ by invoking the Poll CQ verb and either wait for CQ
* to be empty or the number of Poll CQ operations has exceeded
* CQ capacity size;
* - or
* post another WR that completes on the same CQ and wait for this
* WR to return as a WC;
* - and then invoke a Destroy QP or Reset QP.
*
* We use the second option and wait for a completion on the
* rx_drain_qp before destroying QPs attached to our SRQ.
*/
enum ipoib_cm_state {
IPOIB_CM_RX_LIVE,
IPOIB_CM_RX_ERROR, /* Ignored by stale task */
IPOIB_CM_RX_FLUSH /* Last WQE Reached event observed */
};
struct ipoib_cm_rx { struct ipoib_cm_rx {
struct ib_cm_id *id; struct ib_cm_id *id;
struct ib_qp *qp; struct ib_qp *qp;
struct list_head list; struct list_head list;
struct net_device *dev; struct net_device *dev;
unsigned long jiffies; unsigned long jiffies;
enum ipoib_cm_state state;
}; };
struct ipoib_cm_tx { struct ipoib_cm_tx {
...@@ -165,10 +199,16 @@ struct ipoib_cm_dev_priv { ...@@ -165,10 +199,16 @@ struct ipoib_cm_dev_priv {
struct ib_srq *srq; struct ib_srq *srq;
struct ipoib_cm_rx_buf *srq_ring; struct ipoib_cm_rx_buf *srq_ring;
struct ib_cm_id *id; struct ib_cm_id *id;
struct list_head passive_ids; struct ib_qp *rx_drain_qp; /* generates WR described in 10.3.1 */
struct list_head passive_ids; /* state: LIVE */
struct list_head rx_error_list; /* state: ERROR */
struct list_head rx_flush_list; /* state: FLUSH, drain not started */
struct list_head rx_drain_list; /* state: FLUSH, drain started */
struct list_head rx_reap_list; /* state: FLUSH, drain done */
struct work_struct start_task; struct work_struct start_task;
struct work_struct reap_task; struct work_struct reap_task;
struct work_struct skb_task; struct work_struct skb_task;
struct work_struct rx_reap_task;
struct delayed_work stale_task; struct delayed_work stale_task;
struct sk_buff_head skb_queue; struct sk_buff_head skb_queue;
struct list_head start_list; struct list_head start_list;
......
This diff is collapsed.
...@@ -173,7 +173,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca) ...@@ -173,7 +173,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
size = ipoib_sendq_size + ipoib_recvq_size + 1; size = ipoib_sendq_size + ipoib_recvq_size + 1;
ret = ipoib_cm_dev_init(dev); ret = ipoib_cm_dev_init(dev);
if (!ret) if (!ret)
size += ipoib_recvq_size; size += ipoib_recvq_size + 1 /* 1 extra for rx_drain_qp */;
priv->cq = ib_create_cq(priv->ca, ipoib_ib_completion, NULL, dev, size, 0); priv->cq = ib_create_cq(priv->ca, ipoib_ib_completion, NULL, dev, size, 0);
if (IS_ERR(priv->cq)) { if (IS_ERR(priv->cq)) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment