From 6c6341804133ea0a4d4535c621f28f61fe6c29ab Mon Sep 17 00:00:00 2001
From: Doug Oucharek <doug.s.oucharek@intel.com>
Date: Tue, 16 May 2017 16:00:53 -0700
Subject: [PATCH] LU-9500 lnd: Don't Page Align remote_addr with FastReg
Trying to page align the remote_addr for IB_RDMA_WRITE work
requests is triggering "dump_cqe" errors from MOFED 4.x + mlx5.
This patch removes the address masking we were doing with FastReg
which was trying to page align remote_addr values. I am also
removing the setting of "mr->iova" with FastReg as this is being
done in the call to ib_map_mr_sg() and could cause problems.
Signed-off-by: Doug Oucharek <doug.s.oucharek@intel.com>
Change-Id: If35baa467d8d60866f709b5feea7f619063c6da4
Reviewed-on: https://review.whamcloud.com/27149
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Gu Zheng <gzheng@ddn.com>
Reviewed-by: Dmitry Eremin <dmitry.eremin@intel.com>
Reviewed-by: Amir Shehata <amir.shehata@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
---
lnet/klnds/o2iblnd/o2iblnd.c | 6 +++---
lnet/klnds/o2iblnd/o2iblnd.h | 2 +-
lnet/klnds/o2iblnd/o2iblnd_cb.c | 6 ++++--
3 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 298cd9f096..110b6e699f 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1848,7 +1848,7 @@ kiblnd_fmr_pool_unmap(kib_fmr_t *fmr, int status)
int
kiblnd_fmr_pool_map(kib_fmr_poolset_t *fps, kib_tx_t *tx, kib_rdma_desc_t *rd,
- __u32 nob, __u64 iov, kib_fmr_t *fmr)
+ __u32 nob, __u64 iov, kib_fmr_t *fmr, bool *is_fastreg)
{
kib_fmr_pool_t *fpo;
__u64 *pages = tx->tx_pages;
@@ -1868,6 +1868,7 @@ again:
if (fpo->fpo_is_fmr) {
struct ib_pool_fmr *pfmr;
+ *is_fastreg = 0;
spin_unlock(&fps->fps_lock);
if (!tx_pages_mapped) {
@@ -1887,6 +1888,7 @@ again:
}
rc = PTR_ERR(pfmr);
} else {
+ *is_fastreg = 1;
if (!list_empty(&fpo->fast_reg.fpo_pool_list)) {
struct kib_fast_reg_descriptor *frd;
#ifdef HAVE_IB_MAP_MR_SG
@@ -1939,8 +1941,6 @@ again:
return n < 0 ? n : -EINVAL;
}
- mr->iova = iov;
-
wr = &frd->frd_fastreg_wr;
memset(wr, 0, sizeof(*wr));
diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index be6e882f83..c4a4742ac1 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -1187,7 +1187,7 @@ struct list_head *kiblnd_pool_alloc_node(kib_poolset_t *ps);
int kiblnd_fmr_pool_map(kib_fmr_poolset_t *fps, kib_tx_t *tx,
kib_rdma_desc_t *rd, __u32 nob, __u64 iov,
- kib_fmr_t *fmr);
+ kib_fmr_t *fmr, bool *is_fastreg);
void kiblnd_fmr_pool_unmap(kib_fmr_t *fmr, int status);
int kiblnd_tunables_setup(struct lnet_ni *ni);
diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 2ecccbf268..17e2479383 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -569,6 +569,7 @@ kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, __u32 nob)
kib_fmr_poolset_t *fps;
int cpt;
int rc;
+ bool is_fastreg = 0;
LASSERT(tx->tx_pool != NULL);
LASSERT(tx->tx_pool->tpo_pool.po_owner != NULL);
@@ -577,7 +578,7 @@ kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, __u32 nob)
cpt = tx->tx_pool->tpo_pool.po_owner->ps_cpt;
fps = net->ibn_fmr_ps[cpt];
- rc = kiblnd_fmr_pool_map(fps, tx, rd, nob, 0, &tx->fmr);
+ rc = kiblnd_fmr_pool_map(fps, tx, rd, nob, 0, &tx->fmr, &is_fastreg);
if (rc != 0) {
CERROR("Can't map %u pages: %d\n", nob, rc);
return rc;
@@ -586,7 +587,8 @@ kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, __u32 nob)
/* If rd is not tx_rd, it's going to get sent to a peer_ni, who will need
* the rkey */
rd->rd_key = tx->fmr.fmr_key;
- rd->rd_frags[0].rf_addr &= ~hdev->ibh_page_mask;
+ if (!is_fastreg)
+ rd->rd_frags[0].rf_addr &= ~hdev->ibh_page_mask;
rd->rd_frags[0].rf_nob = nob;
rd->rd_nfrags = 1;
--
GitLab