From 8d70042d785e08bb59059ed3ffb0b2f5eb0e3dad Mon Sep 17 00:00:00 2001
From: Amir Shehata <amir.shehata@intel.com>
Date: Tue, 23 May 2017 16:52:37 -0700
Subject: [PATCH] LU-9536 lnet: set LND tunalbes properly
Make sure to set all NIs to the proper LND tunables
specified.
Add ntx tunable to dynamic configuration. This way all
tunables required to tune OPA performance can be configured
via lnetctl, allowing the ability to tune OPA network and IB
network differently
Test-Parameters: trivial
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Change-Id: I015f3f959bd46784d4607bd4259b4640303dc362
Reviewed-on: https://review.whamcloud.com/27263
Tested-by: Jenkins
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Sonia Sharma <sonia.sharma@intel.com>
Reviewed-by: Olaf Weber <olaf.weber@hpe.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
---
lnet/include/lnet/lib-dlc.h | 2 +-
lnet/klnds/o2iblnd/o2iblnd.c | 14 +++++++++-----
lnet/klnds/o2iblnd/o2iblnd_modparams.c | 4 +++-
lnet/utils/lnetconfig/liblnetconfig.c | 7 +++----
lnet/utils/lnetconfig/liblnetconfig_lnd.c | 9 ++++++++-
5 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/lnet/include/lnet/lib-dlc.h b/lnet/include/lnet/lib-dlc.h
index 48662b92ca..4141f7c492 100644
--- a/lnet/include/lnet/lib-dlc.h
+++ b/lnet/include/lnet/lib-dlc.h
@@ -66,7 +66,7 @@ struct lnet_ioctl_config_o2iblnd_tunables {
__u32 lnd_fmr_flush_trigger;
__u32 lnd_fmr_cache;
__u16 lnd_conns_per_peer;
- __u16 pad;
+ __u16 lnd_ntx;
};
struct lnet_lnd_tunables {
diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 12dade5ba4..298cd9f096 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -2305,9 +2305,13 @@ out:
LIBCFS_FREE(tpo, sizeof(kib_tx_pool_t));
}
-static int kiblnd_tx_pool_size(int ncpts)
+static int kiblnd_tx_pool_size(struct lnet_ni *ni, int ncpts)
{
- int ntx = *kiblnd_tunables.kib_ntx / ncpts;
+ struct lnet_ioctl_config_o2iblnd_tunables *tunables;
+ int ntx;
+
+ tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib;
+ ntx = tunables->lnd_ntx / ncpts;
return max(IBLND_TX_POOL, ntx);
}
@@ -2464,10 +2468,10 @@ kiblnd_net_init_pools(kib_net_t *net, struct lnet_ni *ni, __u32 *cpts,
read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
#endif
- if (tunables->lnd_fmr_pool_size < *kiblnd_tunables.kib_ntx / 4) {
+ if (tunables->lnd_fmr_pool_size < tunables->lnd_ntx / 4) {
CERROR("Can't set fmr pool size (%d) < ntx / 4(%d)\n",
tunables->lnd_fmr_pool_size,
- *kiblnd_tunables.kib_ntx / 4);
+ tunables->lnd_ntx / 4);
rc = -EINVAL;
goto failed;
}
@@ -2516,7 +2520,7 @@ kiblnd_net_init_pools(kib_net_t *net, struct lnet_ni *ni, __u32 *cpts,
cpt = (cpts == NULL) ? i : cpts[i];
rc = kiblnd_init_poolset(&net->ibn_tx_ps[cpt]->tps_poolset,
cpt, net, "TX",
- kiblnd_tx_pool_size(ncpts),
+ kiblnd_tx_pool_size(ni, ncpts),
kiblnd_create_tx_pool,
kiblnd_destroy_tx_pool,
kiblnd_tx_init, NULL);
diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index 3ca0689b35..72cb50ecd1 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -162,7 +162,6 @@ kib_tunables_t kiblnd_tunables = {
.kib_cksum = &cksum,
.kib_timeout = &timeout,
.kib_keepalive = &keepalive,
- .kib_ntx = &ntx,
.kib_default_ipif = &ipif_name,
.kib_retry_count = &retry_count,
.kib_rnr_retry_count = &rnr_retry_count,
@@ -290,6 +289,8 @@ kiblnd_tunables_setup(struct lnet_ni *ni)
tunables->lnd_fmr_flush_trigger = fmr_flush_trigger;
if (!tunables->lnd_fmr_cache)
tunables->lnd_fmr_cache = fmr_cache;
+ if (!tunables->lnd_ntx)
+ tunables->lnd_ntx = ntx;
if (!tunables->lnd_conns_per_peer) {
tunables->lnd_conns_per_peer = (conns_per_peer) ?
conns_per_peer : 1;
@@ -308,6 +309,7 @@ kiblnd_tunables_init(void)
default_tunables.lnd_fmr_pool_size = fmr_pool_size;
default_tunables.lnd_fmr_flush_trigger = fmr_flush_trigger;
default_tunables.lnd_fmr_cache = fmr_cache;
+ default_tunables.lnd_ntx = ntx;
default_tunables.lnd_conns_per_peer = conns_per_peer;
return 0;
}
diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c
index 35192e4273..2630d7ed88 100644
--- a/lnet/utils/lnetconfig/liblnetconfig.c
+++ b/lnet/utils/lnetconfig/liblnetconfig.c
@@ -1153,7 +1153,7 @@ lustre_lnet_ioctl_config_ni(struct list_head *intf_list,
list_for_each_entry(intf_descr, intf_list,
intf_on_network) {
- if (i == 0 && tunables != NULL)
+ if (tunables != NULL)
len = sizeof(struct lnet_ioctl_config_ni) +
sizeof(struct lnet_ioctl_config_lnd_tunables);
else
@@ -1163,7 +1163,7 @@ lustre_lnet_ioctl_config_ni(struct list_head *intf_list,
if (!data)
return LUSTRE_CFG_RC_OUT_OF_MEM;
conf = (struct lnet_ioctl_config_ni*) data;
- if (i == 0 && tunables != NULL)
+ if (tunables != NULL)
tun = (struct lnet_ioctl_config_lnd_tunables*)
conf->lic_bulk;
@@ -1197,8 +1197,7 @@ lustre_lnet_ioctl_config_ni(struct list_head *intf_list,
conf->lic_ncpts = count;
- if (i == 0 && tunables != NULL)
- /* TODO put in the LND tunables */
+ if (tunables != NULL)
memcpy(tun, tunables, sizeof(*tunables));
rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_ADD_LOCAL_NI, data);
diff --git a/lnet/utils/lnetconfig/liblnetconfig_lnd.c b/lnet/utils/lnetconfig/liblnetconfig_lnd.c
index ae5d77068c..5aa8787d67 100644
--- a/lnet/utils/lnetconfig/liblnetconfig_lnd.c
+++ b/lnet/utils/lnetconfig/liblnetconfig_lnd.c
@@ -62,6 +62,10 @@ lustre_o2iblnd_show_tun(struct cYAML *lndparams,
lnd_cfg->lnd_fmr_cache) == NULL)
return LUSTRE_CFG_RC_OUT_OF_MEM;
+ if (cYAML_create_number(lndparams, "ntx",
+ lnd_cfg->lnd_ntx) == NULL)
+ return LUSTRE_CFG_RC_OUT_OF_MEM;
+
if (cYAML_create_number(lndparams, "conns_per_peer",
lnd_cfg->lnd_conns_per_peer) == NULL)
return LUSTRE_CFG_RC_OUT_OF_MEM;
@@ -123,7 +127,7 @@ yaml_extract_o2ib_tun(struct cYAML *tree,
struct cYAML *map_on_demand = NULL, *concurrent_sends = NULL;
struct cYAML *fmr_pool_size = NULL, *fmr_cache = NULL;
struct cYAML *fmr_flush_trigger = NULL, *lndparams = NULL;
- struct cYAML *conns_per_peer = NULL;
+ struct cYAML *conns_per_peer = NULL, *ntx = NULL;
lndparams = cYAML_get_object_item(tree, "lnd tunables");
if (!lndparams)
@@ -150,6 +154,9 @@ yaml_extract_o2ib_tun(struct cYAML *tree,
lnd_cfg->lnd_fmr_cache =
(fmr_cache) ? fmr_cache->cy_valueint : 0;
+ ntx = cYAML_get_object_item(lndparams, "ntx");
+ lnd_cfg->lnd_ntx = (ntx) ? ntx->cy_valueint : 0;
+
conns_per_peer = cYAML_get_object_item(lndparams, "conns_per_peer");
lnd_cfg->lnd_conns_per_peer =
(conns_per_peer) ? conns_per_peer->cy_valueint : 1;
--
GitLab