From 27116ee18c04057936837f6f0aef3f4c09c21d78 Mon Sep 17 00:00:00 2001
From: Amir Shehata <amir.shehata@intel.com>
Date: Fri, 5 Jan 2018 11:50:33 -0800
Subject: [PATCH] LU-10394 lnd: default to using MEM_REG
There is a performance drop when using IB_MR_TYPE_SG_GAPS. To
mitigate this, we added a module parameter, use_fastreg_gaps, which
defaults to 0. When allocating the memory region if this parameter
is set to 1 and the hw has gaps support then use it and output
a warning that performance may drop. Otherwise always use
IB_MR_TYPE_MEM_REG.
Test-Parameters: trivial
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Change-Id: I08a8b72756b9b5b5bcb391bf3e979f6d28eb5cbb
Reviewed-on: https://review.whamcloud.com/30749
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Reviewed-by: Doug Oucharek <dougso@me.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
---
lnet/klnds/o2iblnd/o2iblnd.c | 7 +++++--
lnet/klnds/o2iblnd/o2iblnd.h | 1 +
lnet/klnds/o2iblnd/o2iblnd_cb.c | 3 ++-
lnet/klnds/o2iblnd/o2iblnd_modparams.c | 5 +++++
4 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 798132d377..fc1ce478ca 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1592,14 +1592,17 @@ static int kiblnd_alloc_freg_pool(kib_fmr_poolset_t *fps, kib_fmr_pool_t *fpo,
*/
frd->frd_mr = ib_alloc_mr(fpo->fpo_hdev->ibh_pd,
#ifdef IB_MR_TYPE_SG_GAPS
- (dev_caps &
- IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT) ?
+ ((*kiblnd_tunables.kib_use_fastreg_gaps == 1) &&
+ (dev_caps & IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT)) ?
IB_MR_TYPE_SG_GAPS :
IB_MR_TYPE_MEM_REG,
#else
IB_MR_TYPE_MEM_REG,
#endif
LNET_MAX_PAYLOAD/PAGE_SIZE);
+ if ((*kiblnd_tunables.kib_use_fastreg_gaps == 1) &&
+ (dev_caps & IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT))
+ CWARN("using IB_MR_TYPE_SG_GAPS, expect a performance drop\n");
#endif
if (IS_ERR(frd->frd_mr)) {
rc = PTR_ERR(frd->frd_mr);
diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index 7d2e6e032c..ccc859685f 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -112,6 +112,7 @@ typedef struct
/* # threads on each CPT */
int *kib_nscheds;
int *kib_wrq_sge; /* # sg elements per wrq */
+ int *kib_use_fastreg_gaps; /* enable discontiguous fastreg fragment support */
} kib_tunables_t;
extern kib_tunables_t kiblnd_tunables;
diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index faa3170677..b96131a4c7 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -567,7 +567,8 @@ kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, __u32 nob)
if ((dev->ibd_dev_caps & IBLND_DEV_CAPS_FASTREG_ENABLED) &&
!(dev->ibd_dev_caps & IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT) &&
tx->tx_gaps) {
- CERROR("Using FastReg with no GAPS support, but tx has gaps\n");
+ CERROR("Using FastReg with no GAPS support, but tx has gaps. "
+ "Try setting use_fastreg_gaps to 1\n");
return -EPROTONOSUPPORT;
}
diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index 85c66fddaf..2224b04a50 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -110,6 +110,10 @@ static int concurrent_sends;
module_param(concurrent_sends, int, 0444);
MODULE_PARM_DESC(concurrent_sends, "send work-queue sizing (obsolete)");
+static int use_fastreg_gaps;
+module_param(use_fastreg_gaps, int, 0444);
+MODULE_PARM_DESC(use_fastreg_gaps, "Enable discontiguous fastreg fragment support. Expect performance drop");
+
/*
* map_on_demand is a flag used to determine if we can use FMR or FastReg.
* This is applicable for kernels which support global memory regions. For
@@ -196,6 +200,7 @@ kib_tunables_t kiblnd_tunables = {
.kib_use_priv_port = &use_privileged_port,
.kib_nscheds = &nscheds,
.kib_wrq_sge = &wrq_sge,
+ .kib_use_fastreg_gaps = &use_fastreg_gaps,
};
static struct lnet_ioctl_config_o2iblnd_tunables default_tunables;
--
GitLab