From 295e85c302f51f282374ad9c197caa1abd9c2b8b Mon Sep 17 00:00:00 2001
From: scjody <scjody>
Date: Tue, 24 Jul 2007 21:34:42 +0000
Subject: [PATCH] Branch b1_6
Description: Async OSC create to avoid the blocking unnecessarily.
Details : If a OST has no remain object, system will block on the creating
when need to create a new object on this OST. Now, ways use
pre-created objects when available, instead of blocking on an
empty osc while others are not empty. If we must block, we block
for the shortest possible period of time.
Patch by shadow.
b=4900
i=wangdi
i=nathan
---
lustre/ChangeLog | 9 +++
lustre/include/obd.h | 1 +
lustre/include/obd_class.h | 12 ++++
lustre/include/obd_support.h | 1 +
lustre/lov/lov_internal.h | 2 +
lustre/lov/lov_obd.c | 2 +-
lustre/lov/lov_qos.c | 100 +++++++++++++++++++++++++------
lustre/obdclass/lprocfs_status.c | 1 +
lustre/obdfilter/filter.c | 2 +
lustre/osc/osc_create.c | 33 ++++++++++
lustre/osc/osc_internal.h | 1 +
lustre/osc/osc_request.c | 1 +
lustre/tests/sanity.sh | 44 +++++++++++++-
13 files changed, 188 insertions(+), 21 deletions(-)
diff --git a/lustre/ChangeLog b/lustre/ChangeLog
index fa1832159c..777e438446 100644
--- a/lustre/ChangeLog
+++ b/lustre/ChangeLog
@@ -580,6 +580,15 @@ Frequency : rare
Description: Using wrong pointer in osc_brw_prep_request
Details : Access to array[-1] can produce panic if kernel compiled with
CONFIG_PAGE_ALLOC enabled
+
+Severity : enhancement
+Bugzilla : 4900
+Description: Async OSC create to avoid the blocking unnecessarily.
+Details : If a OST has no remain object, system will block on the creating
+ when need to create a new object on this OST. Now, ways use
+ pre-created objects when available, instead of blocking on an
+ empty osc while others are not empty. If we must block, we block
+ for the shortest possible period of time.
--------------------------------------------------------------------------------
diff --git a/lustre/include/obd.h b/lustre/include/obd.h
index 7484762a34..83d0c52633 100644
--- a/lustre/include/obd.h
+++ b/lustre/include/obd.h
@@ -858,6 +858,7 @@ struct obd_ops {
struct lov_stripe_md *mem_tgt);
int (*o_preallocate)(struct lustre_handle *, obd_count *req,
obd_id *ids);
+ int (*o_precreate)(struct obd_export *exp, int need_create);
int (*o_create)(struct obd_export *exp, struct obdo *oa,
struct lov_stripe_md **ea, struct obd_trans_info *oti);
int (*o_destroy)(struct obd_export *exp, struct obdo *oa,
diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h
index d9deddb691..cfb217eab2 100644
--- a/lustre/include/obd_class.h
+++ b/lustre/include/obd_class.h
@@ -476,6 +476,18 @@ static inline int obd_checkmd(struct obd_export *exp,
RETURN(rc);
}
+static inline int obd_precreate(struct obd_export *exp, int need_create)
+{
+ int rc;
+ ENTRY;
+
+ EXP_CHECK_OP(exp, precreate);
+ OBD_COUNTER_INCREMENT(exp->exp_obd, precreate);
+
+ rc = OBP(exp->exp_obd, precreate)(exp, need_create);
+ RETURN(rc);
+}
+
static inline int obd_create(struct obd_export *exp, struct obdo *obdo,
struct lov_stripe_md **ea,
struct obd_trans_info *oti)
diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index 5f987d4399..40aa2b3f2d 100644
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -103,6 +103,7 @@ extern int obd_race_state;
#define OBD_FAIL_MDS_RESEND 0x136
#define OBD_FAIL_MDS_LLOG_CREATE_FAILED 0x137
#define OBD_FAIL_MDS_LOV_SYNC_RACE 0x138
+#define OBD_FAIL_MDS_OSC_PRECREATE 0x139
#define OBD_FAIL_OST 0x200
#define OBD_FAIL_OST_CONNECT_NET 0x201
diff --git a/lustre/lov/lov_internal.h b/lustre/lov/lov_internal.h
index 85c80cd10b..565cc6f1c0 100644
--- a/lustre/lov/lov_internal.h
+++ b/lustre/lov/lov_internal.h
@@ -144,6 +144,8 @@ int lov_stripe_intersects(struct lov_stripe_md *lsm, int stripeno,
int lov_stripe_number(struct lov_stripe_md *lsm, obd_off lov_off);
/* lov_qos.c */
+#define LOV_USES_ASSIGNED_STRIPE 0
+#define LOV_USES_DEFAULT_STRIPE 1
int qos_add_tgt(struct obd_device *obd, __u32 index);
int qos_del_tgt(struct obd_device *obd, __u32 index);
void qos_shrink_lsm(struct lov_request_set *set);
diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c
index 7f5970c9e7..0167df50e9 100644
--- a/lustre/lov/lov_obd.c
+++ b/lustre/lov/lov_obd.c
@@ -513,7 +513,7 @@ static int lov_add_target(struct obd_device *obd, struct obd_uuid *uuidp,
CDEBUG(D_CONFIG, "tgts: %p size: %d\n",
lov->lov_tgts, lov->lov_tgt_size);
- }
+ }
OBD_ALLOC_PTR(tgt);
diff --git a/lustre/lov/lov_qos.c b/lustre/lov/lov_qos.c
index 2183ca27f9..f0d3eb7935 100644
--- a/lustre/lov/lov_qos.c
+++ b/lustre/lov/lov_qos.c
@@ -485,24 +485,31 @@ int qos_remedy_create(struct lov_request_set *set, struct lov_request *req)
#define LOV_CREATE_RESEED_MULT 4
#define LOV_CREATE_RESEED_MIN 1000
/* Allocate objects on osts with round-robin algorithm */
-static int alloc_rr(struct lov_obd *lov, int *idx_arr, int *stripe_cnt)
+static int alloc_rr(struct lov_obd *lov, int *idx_arr, int *stripe_cnt_orig, int flags)
{
- unsigned array_idx, ost_count = lov->desc.ld_tgt_count;
+ unsigned array_idx, array_idx_temp, ost_count = lov->desc.ld_tgt_count;
unsigned ost_active_count = lov->desc.ld_active_tgt_count;
- int i, *idx_pos = idx_arr;
+ int i, *idx_pos;
__u32 ost_idx;
+ int first_pass = 1, ost_start_idx_temp;
+ int want_level = 0;
+ int stripe_cnt;
ENTRY;
i = qos_calc_rr(lov);
if (i)
RETURN(i);
+ stripe_cnt = flags == LOV_USES_DEFAULT_STRIPE ?
+ *stripe_cnt_orig - (*stripe_cnt_orig/4) :
+ *stripe_cnt_orig;
+
if (--lov->lov_start_count <= 0) {
lov->lov_start_idx = ll_rand() % ost_count;
lov->lov_start_count =
(LOV_CREATE_RESEED_MIN / max(ost_active_count, 1U) +
LOV_CREATE_RESEED_MULT) * max(ost_active_count, 1U);
- } else if (*stripe_cnt >= ost_active_count ||
+ } else if (stripe_cnt >= ost_active_count ||
lov->lov_start_idx > ost_count) {
/* If we have allocated from all of the OSTs, slowly
precess the next start */
@@ -512,11 +519,16 @@ static int alloc_rr(struct lov_obd *lov, int *idx_arr, int *stripe_cnt)
array_idx = (lov->lov_start_idx + lov->lov_offset_idx) % ost_count;
#ifdef QOS_DEBUG
CDEBUG(D_QOS, "want %d startidx %d startcnt %d offset %d arrayidx %d\n",
- *stripe_cnt, lov->lov_start_idx, lov->lov_start_count,
+ stripe_cnt, lov->lov_start_idx, lov->lov_start_count,
lov->lov_offset_idx, array_idx);
#endif
-
down_read(&lov->lov_qos.lq_rw_sem);
+ ost_start_idx_temp = lov->lov_start_idx;
+ array_idx_temp = array_idx;
+
+repeat_find :
+ idx_pos = idx_arr;
+
for (i = 0; i < ost_count; i++, array_idx=(array_idx + 1) % ost_count) {
++lov->lov_start_idx;
ost_idx = lov->lov_qos.lq_rr_array[array_idx];
@@ -527,18 +539,37 @@ static int alloc_rr(struct lov_obd *lov, int *idx_arr, int *stripe_cnt)
lov->lov_tgts[ost_idx]->ltd_active : 0,
idx_pos - idx_arr, array_idx, ost_idx);
#endif
- if ((ost_idx == LOV_QOS_EMPTY) || !lov->lov_tgts[ost_idx] ||
+ if ((ost_idx == LOV_QOS_EMPTY) || !lov->lov_tgts[ost_idx] ||
!lov->lov_tgts[ost_idx]->ltd_active)
continue;
+
+ /* Fail Check before osc_precreate() is called
+ so we can only 'fail' single OSC. */
+ if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OSC_PRECREATE) && ost_idx == 0)
+ continue;
+
+ /* the osc_precreate() will be called */
+ if (obd_precreate(lov->lov_tgts[ost_idx]->ltd_exp, first_pass) > want_level)
+ continue;
+
*idx_pos = ost_idx;
idx_pos++;
/* We have enough stripes */
- if (idx_pos - idx_arr == *stripe_cnt)
+ if (idx_pos - idx_arr == *stripe_cnt_orig)
break;
}
+ if (first_pass && (idx_pos - idx_arr < stripe_cnt)) {
+ /* not send precreate and skip only failed ost */
+ first_pass = 0;
+ want_level = 1;
+ lov->lov_start_idx = ost_start_idx_temp;
+ array_idx = array_idx_temp;
+ goto repeat_find;
+ }
+
up_read(&lov->lov_qos.lq_rw_sem);
- *stripe_cnt = idx_pos - idx_arr;
+ *stripe_cnt_orig = idx_pos - idx_arr;
RETURN(0);
}
@@ -548,20 +579,39 @@ static int alloc_specific(struct lov_obd *lov, struct lov_stripe_md *lsm,
{
unsigned ost_idx, ost_count = lov->desc.ld_tgt_count;
int i, *idx_pos = idx_arr;
+ int first_pass = 1;
+ int want_level = 0;
ENTRY;
+repeat_find:
ost_idx = lsm->lsm_oinfo[0]->loi_ost_idx;
for (i = 0; i < ost_count; i++, ost_idx = (ost_idx + 1) % ost_count) {
if (!lov->lov_tgts[ost_idx] ||
!lov->lov_tgts[ost_idx]->ltd_active) {
continue;
}
+
+ /* Fail Check before osc_precreate() is called
+ so we can only 'fail' single OSC. */
+ if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OSC_PRECREATE) && ost_idx == 0)
+ continue;
+
+ /* the osc_precreate() will be called */
+ if (obd_precreate(lov->lov_tgts[ost_idx]->ltd_exp, first_pass) > want_level)
+ continue;
+
*idx_pos = ost_idx;
idx_pos++;
/* got enough ost */
if (idx_pos - idx_arr == lsm->lsm_stripe_count)
RETURN(0);
}
+ if (first_pass) {
+ first_pass = 0;
+ want_level = 1;
+ goto repeat_find;
+ }
+
/* If we were passed specific striping params, then a failure to
* meet those requirements is an error, since we can't reallocate
* that memory (it might be part of a larger array or something).
@@ -577,7 +627,8 @@ static int alloc_specific(struct lov_obd *lov, struct lov_stripe_md *lsm,
- free space
- network resources (shared OSS's)
*/
-static int alloc_qos(struct obd_export *exp, int *idx_arr, int *stripe_cnt)
+static int alloc_qos(struct obd_export *exp, int *idx_arr, int *stripe_cnt,
+ int flags)
{
struct lov_obd *lov = &exp->exp_obd->u.lov;
static time_t last_warn = 0;
@@ -628,6 +679,14 @@ static int alloc_qos(struct obd_export *exp, int *idx_arr, int *stripe_cnt)
continue;
}
+ /* Fail Check before osc_precreate() is called
+ so we can only 'fail' single OSC. */
+ if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OSC_PRECREATE) && i == 0)
+ continue;
+
+ if (obd_precreate(lov->lov_tgts[i]->ltd_exp, 1) == 2)
+ continue;
+
lov->lov_tgts[i]->ltd_qos.ltq_usable = 1;
qos_calc_weight(lov, i);
total_bavail += bavail;
@@ -685,6 +744,7 @@ static int alloc_qos(struct obd_export *exp, int *idx_arr, int *stripe_cnt)
if (!lov->lov_tgts[i] ||
!lov->lov_tgts[i]->ltd_qos.ltq_usable)
continue;
+
cur_weight += lov->lov_tgts[i]->ltd_qos.ltq_weight;
if (cur_weight >= rand) {
#ifdef QOS_DEBUG
@@ -709,7 +769,7 @@ out:
up_write(&lov->lov_qos.lq_rw_sem);
if (rc == -EAGAIN)
- rc = alloc_rr(lov, idx_arr, stripe_cnt);
+ rc = alloc_rr(lov, idx_arr, stripe_cnt, flags);
lov_putref(exp->exp_obd);
RETURN(rc);
@@ -717,7 +777,7 @@ out:
/* return new alloced stripe count on success */
static int alloc_idx_array(struct obd_export *exp, struct lov_stripe_md *lsm,
- int newea, int **idx_arr, int *arr_cnt)
+ int newea, int **idx_arr, int *arr_cnt, int flags)
{
struct lov_obd *lov = &exp->exp_obd->u.lov;
int stripe_cnt = lsm->lsm_stripe_count;
@@ -734,7 +794,7 @@ static int alloc_idx_array(struct obd_export *exp, struct lov_stripe_md *lsm,
if (newea ||
lsm->lsm_oinfo[0]->loi_ost_idx >= lov->desc.ld_tgt_count)
- rc = alloc_qos(exp, tmp_arr, &stripe_cnt);
+ rc = alloc_qos(exp, tmp_arr, &stripe_cnt, flags);
else
rc = alloc_specific(lov, lsm, tmp_arr);
@@ -762,13 +822,14 @@ int qos_prep_create(struct obd_export *exp, struct lov_request_set *set)
struct obdo *src_oa = set->set_oi->oi_oa;
struct obd_trans_info *oti = set->set_oti;
int i, stripes, rc = 0, newea = 0;
+ int flag = LOV_USES_ASSIGNED_STRIPE;
int *idx_arr, idx_cnt = 0;
ENTRY;
LASSERT(src_oa->o_valid & OBD_MD_FLID);
if (set->set_oi->oi_md == NULL) {
- int stripe_cnt = lov_get_stripecnt(lov, 0);
+ int stripes_def = lov_get_stripecnt(lov, 0);
/* If the MDS file was truncated up to some size, stripe over
* enough OSTs to allow the file to be created at that size.
@@ -791,10 +852,11 @@ int qos_prep_create(struct obd_export *exp, struct lov_request_set *set)
}
lov_putref(exp->exp_obd);
- if (stripes < stripe_cnt)
- stripes = stripe_cnt;
+ if (stripes < stripes_def)
+ stripes = stripes_def;
} else {
- stripes = stripe_cnt;
+ flag = LOV_USES_DEFAULT_STRIPE;
+ stripes = stripes_def;
}
rc = lov_alloc_memmd(&set->set_oi->oi_md, stripes,
@@ -803,8 +865,8 @@ int qos_prep_create(struct obd_export *exp, struct lov_request_set *set)
LOV_MAGIC);
if (rc < 0)
GOTO(out_err, rc);
- rc = 0;
newea = 1;
+ rc = 0;
}
lsm = set->set_oi->oi_md;
@@ -816,7 +878,7 @@ int qos_prep_create(struct obd_export *exp, struct lov_request_set *set)
lsm->lsm_pattern = lov->desc.ld_pattern;
}
- stripes = alloc_idx_array(exp, lsm, newea, &idx_arr, &idx_cnt);
+ stripes = alloc_idx_array(exp, lsm, newea, &idx_arr, &idx_cnt, flag);
if (stripes <= 0)
GOTO(out_err, rc = stripes ? stripes : -EIO);
LASSERTF(stripes <= lsm->lsm_stripe_count,"requested %d allocated %d\n",
diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c
index 29bc50821c..1b2ba7ac9a 100644
--- a/lustre/obdclass/lprocfs_status.c
+++ b/lustre/obdclass/lprocfs_status.c
@@ -847,6 +847,7 @@ void lprocfs_init_ops_stats(int num_private_stats, struct lprocfs_stats *stats)
LPROCFS_OBD_OP_INIT(num_private_stats, stats, unpackmd);
LPROCFS_OBD_OP_INIT(num_private_stats, stats, checkmd);
LPROCFS_OBD_OP_INIT(num_private_stats, stats, preallocate);
+ LPROCFS_OBD_OP_INIT(num_private_stats, stats, precreate);
LPROCFS_OBD_OP_INIT(num_private_stats, stats, create);
LPROCFS_OBD_OP_INIT(num_private_stats, stats, destroy);
LPROCFS_OBD_OP_INIT(num_private_stats, stats, setattr);
diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c
index cc5b9619df..393c2bdb22 100644
--- a/lustre/obdfilter/filter.c
+++ b/lustre/obdfilter/filter.c
@@ -2798,6 +2798,8 @@ static int filter_precreate(struct obd_device *obd, struct obdo *oa,
LASSERT(down_trylock(&filter->fo_create_lock) != 0);
+ OBD_FAIL_TIMEOUT(OBD_FAIL_TGT_DELAY_PRECREATE, obd_timeout / 2);
+
if ((oa->o_valid & OBD_MD_FLFLAGS) &&
(oa->o_flags & OBD_FL_RECREATE_OBJS)) {
recreate_obj = 1;
diff --git a/lustre/osc/osc_create.c b/lustre/osc/osc_create.c
index 8a5a0ed20b..e8fc4a9349 100644
--- a/lustre/osc/osc_create.c
+++ b/lustre/osc/osc_create.c
@@ -239,6 +239,39 @@ int oscc_recovering(struct osc_creator *oscc)
return recov;
}
+/* decide if the OST has remaining object, return value :
+ 0 : the OST has remaining object, and don't need to do precreate.
+ 1 : the OST has no remaining object, and will send a RPC for precreate.
+ 2 : the OST has no remaining object, and will not get any for
+ a potentially very long time
+ */
+int osc_precreate(struct obd_export *exp, int need_create)
+{
+ struct osc_creator *oscc = &exp->exp_obd->u.cli.cl_oscc;
+ struct obd_import *imp = exp->exp_imp_reverse;
+ ENTRY;
+
+ LASSERT(oscc != NULL);
+ if (imp != NULL && imp->imp_deactive)
+ RETURN(2);
+
+ if (oscc->oscc_last_id < oscc->oscc_next_id) {
+ if (oscc->oscc_flags & OSCC_FLAG_NOSPC ||
+ oscc_recovering(oscc))
+ RETURN(2);
+
+ if (oscc->oscc_flags & OSCC_FLAG_CREATING)
+ RETURN(1);
+
+ if (!need_create)
+ RETURN(1);
+
+ oscc_internal_create(oscc);
+ RETURN(1);
+ }
+ RETURN(0);
+}
+
int osc_create(struct obd_export *exp, struct obdo *oa,
struct lov_stripe_md **ea, struct obd_trans_info *oti)
{
diff --git a/lustre/osc/osc_internal.h b/lustre/osc/osc_internal.h
index 7fd68ba316..4ae9720639 100644
--- a/lustre/osc/osc_internal.h
+++ b/lustre/osc/osc_internal.h
@@ -54,6 +54,7 @@ struct osc_cache_waiter {
#define OSCC_FLAG_LOW 0x10
#define OSCC_FLAG_EXITING 0x20
+int osc_precreate(struct obd_export *exp, int need_create);
int osc_create(struct obd_export *exp, struct obdo *oa,
struct lov_stripe_md **ea, struct obd_trans_info *oti);
int osc_real_create(struct obd_export *exp, struct obdo *oa,
diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index 3868c5e56a..9c9adc459d 100644
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -3666,6 +3666,7 @@ struct obd_ops osc_obd_ops = {
.o_statfs_async = osc_statfs_async,
.o_packmd = osc_packmd,
.o_unpackmd = osc_unpackmd,
+ .o_precreate = osc_precreate,
.o_create = osc_create,
.o_destroy = osc_destroy,
.o_getattr = osc_getattr,
diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index a55f51b144..e88f38f435 100644
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -1141,6 +1141,48 @@ test_27x() { # bug 10997
}
run_test 27x "check lfs setstripe -c -s -i options ============="
+test_27u() { # bug 4900
+ [ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && echo "skip $TESTNAME" && return
+ #define OBD_FAIL_MDS_OSC_PRECREATE 0x139
+
+ sysctl -w lustre.fail_loc=0x139
+ mkdir -p $DIR/d27u
+ createmany -o $DIR/d27u/t- 1000
+ sysctl -w lustre.fail_loc=0
+
+ $LFS getstripe $DIR/d27u > $TMP/files
+ OBJS=`cat $TMP/files | awk -vobjs=0 '($1 == 0) { objs += 1 } END { print objs;}'`
+ unlinkmany $DIR/d27u/t- 1000
+ [ $OBJS -gt 0 ] && \
+ error "Found $OBJS objects were created on OST-0" || pass
+}
+run_test 27u "skip object creation on OSC w/o objects =========="
+
+test_27v() { # bug 4900
+ [ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && echo "skip $TESTNAME" && return
+ exhaust_all_precreations
+
+ mkdir -p $DIR/$tdir
+ lfs setstripe $DIR/$tdir 0 -1 1 # 1 stripe / file
+
+ touch $DIR/$tdir/$tfile
+ #define OBD_FAIL_TGT_DELAY_PRECREATE 0x705
+ sysctl -w lustre.fail_loc=0x705
+ START=`date +%s`
+ for F in `seq 1 32`; do
+ touch $DIR/$tdir/$tfile.$F
+ done
+ sysctl -w lustre.fail_loc=0
+
+ FINISH=`date +%s`
+ TIMEOUT=`sysctl -n lustre.timeout`
+ [ $((FINISH - START)) -ge $((TIMEOUT / 2)) ] && \
+ error "$FINISH - $START >= $TIMEOUT / 2"
+
+ reset_enospc
+}
+run_test 27v "skip object creation on slow OST ================="
+
test_28() {
mkdir $DIR/d28
$CREATETEST $DIR/d28/ct || error
@@ -2670,7 +2712,7 @@ test_65j() { # bug6367
cleanup -f || error "failed to unmount"
setup
fi
- $SETSTRIPE -d $MOUNT
+ $SETSTRIPE -d $MOUNT || error "setstripe failed"
}
run_test 65j "set default striping on root directory (bug 6367)="
--
GitLab