From 99bf8fb2147b61f11e61d3330b61147eba48d5fd Mon Sep 17 00:00:00 2001
From: yury <yury>
Date: Mon, 22 Aug 2005 11:57:40 +0000
Subject: [PATCH] - added more debug in osc. Currently two ideas are working
for checks why osc does not give desired band:
(1) cache size is choosen kind of cumbersome way and not enough, what makes writing thread waiting for cache in osc_enter_cache(). Cache size curremtly does not depend on how good wire is able to send RPCs to remote node what seems to be an artificial limit in some cases. If this is so, the fix would be to adjust cache size depending on much it is needed. This however does not explain why second subtest suffers from slowdown whereas first one does not.
(2) due to some reasons (limited cache or limited grant from OST) many async RPCs are falled back to synchronous IO what makes substential slowdown.
Added debug on real cluster should show what is going on there. Idea that issue on server has a little chance to be true, as server uses direct IO and there is almost not ability to be slowed down due to some cache issues, etc.
---
lustre/include/linux/obd.h | 3 ++-
lustre/ldlm/ldlm_lib.c | 26 ++++++++++++++------------
lustre/osc/osc_request.c | 10 +++++++---
3 files changed, 23 insertions(+), 16 deletions(-)
diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h
index a6a7146ea6..4b00043f32 100644
--- a/lustre/include/linux/obd.h
+++ b/lustre/include/linux/obd.h
@@ -302,7 +302,6 @@ struct mds_server_data;
#define OSC_MAX_DIRTY_DEFAULT (4*OSC_MAX_RIF_DEFAULT*PTLRPC_MAX_BRW_SIZE>>20)
#define OSC_MAX_DIRTY_MB_MAX 512 /* totally arbitrary */
-
struct mdc_rpc_lock;
struct client_obd {
struct obd_import *cl_import;
@@ -373,6 +372,8 @@ struct client_obd {
unsigned long cl_dirty_dmax;
unsigned long cl_dirty_dmin;
+
+ unsigned long cl_sync_rpcs;
};
/* Like a client, with some hangers-on. Keep mc_client_obd first so that we
diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c
index c4f5f60642..e16c61a7cb 100644
--- a/lustre/ldlm/ldlm_lib.c
+++ b/lustre/ldlm/ldlm_lib.c
@@ -238,6 +238,7 @@ int client_obd_setup(struct obd_device *obddev, obd_count len, void *buf)
cli->cl_dirty = 0;
cli->cl_avail_grant = 0;
+
/* FIXME: should limit this for the sum of all cl_dirty_max */
cli->cl_dirty_max = OSC_MAX_DIRTY_DEFAULT * 1024 * 1024;
if (cli->cl_dirty_max >> PAGE_SHIFT > num_physpages / 8)
@@ -269,6 +270,7 @@ int client_obd_setup(struct obd_device *obddev, obd_count len, void *buf)
cli->cl_dirty_av = 0;
cli->cl_dirty_dmax = 0;
cli->cl_dirty_dmin = 0;
+ cli->cl_sync_rpcs = 0;
if (num_physpages >> (20 - PAGE_SHIFT) <= 128) { /* <= 128 MB */
cli->cl_max_pages_per_rpc = PTLRPC_MAX_BRW_PAGES / 4;
@@ -373,6 +375,7 @@ int client_obd_cleanup(struct obd_device *obddev, int flags)
if (!cli->cl_import)
RETURN(-EINVAL);
+
if (cli->cl_mgmtcli_obd) {
mgmtcli_deregister_for_events_t dereg_f;
@@ -383,10 +386,10 @@ int client_obd_cleanup(struct obd_device *obddev, int flags)
if (cli->cl_write_gaps) {
CWARN("%s: [writes num: %lu, reads num: %lu]: %lu write gaps: %lu "
- "av. (usec), %lu total (usec)\n", obddev->obd_name,
- cli->cl_write_num, cli->cl_read_num, cli->cl_write_gaps,
- cli->cl_write_gap_sum / cli->cl_write_gaps,
- cli->cl_write_gap_sum);
+ "av. (usec), %lu total (usec), %lu rpcs falled back to sync\n",
+ obddev->obd_name, cli->cl_write_num, cli->cl_read_num,
+ cli->cl_write_gaps, cli->cl_write_gap_sum / cli->cl_write_gaps,
+ cli->cl_write_gap_sum, cli->cl_sync_rpcs);
}
if (cli->cl_cache_wait_num) {
@@ -396,14 +399,13 @@ int client_obd_cleanup(struct obd_device *obddev, int flags)
}
if (cli->cl_dirty_av) {
- CWARN("%s: pipe loading av. %lu (b), max pipe space %lu (b), pipe "
- "loading ratio %lu%%\n", obddev->obd_name, cli->cl_dirty_av,
- cli->cl_dirty_max, (cli->cl_dirty_av * 100) / cli->cl_dirty_max);
- }
-
- if (cli->cl_dirty_dmax) {
- CWARN("%s: pipe dirty max %lu (b), pipe dirty min %lu (b)\n",
- obddev->obd_name, cli->cl_dirty_dmax, cli->cl_dirty_dmin);
+ CWARN("%s: pipe loading av. %lu (b), max pipe room %lu (b), pipe "
+ "loading av. ratio %lu%%, pipe dirty max %lu (b), pipe dirty "
+ "min %lu (b), pipe loading max ratio %lu%%\n", obddev->obd_name,
+ cli->cl_dirty_av, cli->cl_dirty_max,
+ (cli->cl_dirty_av * 100) / cli->cl_dirty_max,
+ cli->cl_dirty_dmax, cli->cl_dirty_dmin,
+ (cli->cl_dirty_dmax * 100) / cli->cl_dirty_max);
}
/* Here we try to drop the security structure after destroy import,
diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index b2463ac83a..de0f5290bf 100644
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -1861,13 +1861,14 @@ static int osc_enter_cache(struct client_obd *cli, struct lov_oinfo *loi,
struct osc_cache_waiter ocw;
struct l_wait_info lwi = { 0 };
struct timeval start, stop;
+ ENTRY;
CDEBUG(D_CACHE, "dirty: %ld dirty_max: %ld dropped: %lu grant: %lu\n",
cli->cl_dirty, cli->cl_dirty_max, cli->cl_lost_grant,
cli->cl_avail_grant);
if (cli->cl_dirty_max < PAGE_SIZE)
- return(-EDQUOT);
+ GOTO(out, -EDQUOT);
if (~0ul - cli->cl_dirty_sum <= cli->cl_dirty) {
cli->cl_dirty_av = (cli->cl_dirty_av +
@@ -1892,7 +1893,7 @@ static int osc_enter_cache(struct client_obd *cli, struct lov_oinfo *loi,
cli->cl_avail_grant >= PAGE_SIZE) {
/* account for ourselves */
osc_consume_write_grant(cli, oap);
- return(0);
+ RETURN(0);
}
/* Make sure that there are write rpcs in flight to wait for. This
@@ -1925,7 +1926,10 @@ static int osc_enter_cache(struct client_obd *cli, struct lov_oinfo *loi,
RETURN(ocw.ocw_rc);
}
- RETURN(-EDQUOT);
+ EXIT;
+out:
+ cli->cl_sync_rpcs++;
+ return -EDQUOT;
}
/* the companion to enter_cache, called when an oap is no longer part of the
--
GitLab