From 9d3ad9fba3e244735f0994c3910e9de00e2a7f4d Mon Sep 17 00:00:00 2001 From: bobijam <bobijam> Date: Mon, 27 Oct 2008 02:15:02 +0000 Subject: [PATCH] Branch HEAD b=11089 i=nathan.rutman i=adilger per-nid client statistics for MDT. --- lustre/include/obd.h | 17 +++++---- lustre/include/obd_class.h | 5 ++- lustre/ldlm/ldlm_lib.c | 34 ++++++++--------- lustre/mdt/Makefile.in | 2 +- lustre/mdt/mdt_fs.c | 78 ++++++++++++++++++++++++++++++++++++++ lustre/mdt/mdt_handler.c | 19 +++++++++- lustre/mdt/mdt_internal.h | 8 +++- lustre/mdt/mdt_recovery.c | 1 + lustre/mgs/mgs_fs.c | 6 +-- lustre/mgs/mgs_handler.c | 3 +- lustre/obdfilter/filter.c | 31 ++++++++++++--- lustre/osc/osc_request.c | 5 ++- lustre/ptlrpc/import.c | 2 +- 13 files changed, 167 insertions(+), 44 deletions(-) create mode 100644 lustre/mdt/mdt_fs.c diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 76b354fe04..b42e2c3736 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -510,7 +510,7 @@ struct client_obd { __u32 cl_supp_cksum_types; /* checksum algorithm to be used */ cksum_type_t cl_cksum_type; - + /* also protected by the poorly named _loi_list_lock lock above */ struct osc_async_rc cl_ar; @@ -1020,7 +1020,7 @@ struct obd_device { /* XXX encapsulate all this recovery data into one struct */ svc_handler_t obd_recovery_handler; pid_t obd_processing_task; - + int obd_max_recoverable_clients; int obd_connected_clients; int obd_recoverable_clients; @@ -1036,7 +1036,7 @@ struct obd_device { time_t obd_recovery_end; /* seconds, for lprocfs_status */ time_t obd_recovery_max_time; /* seconds, bz13079 */ int obd_recovery_timeout; - + /* new recovery stuff from CMD2 */ struct target_recovery_data obd_recovery_data; int obd_replayed_locks; @@ -1073,8 +1073,8 @@ struct obd_device { atomic_t obd_evict_inprogress; cfs_waitq_t obd_evict_inprogress_waitq; - /** - * Ldlm pool part. Save last calculated SLV and Limit. + /** + * Ldlm pool part. Save last calculated SLV and Limit. */ rwlock_t obd_pool_lock; int obd_pool_limit; @@ -1137,7 +1137,7 @@ static inline int it_to_lock_mode(struct lookup_intent *it) return LCK_CW; else if (it->it_op & (IT_READDIR | IT_GETATTR | IT_OPEN | IT_LOOKUP)) return LCK_CR; - + LASSERTF(0, "Invalid it_op: %d\n", it->it_op); return -EINVAL; } @@ -1233,7 +1233,8 @@ struct obd_ops { int (*o_reconnect)(const struct lu_env *env, struct obd_export *exp, struct obd_device *src, struct obd_uuid *cluuid, - struct obd_connect_data *ocd); + struct obd_connect_data *ocd, + void *localdata); int (*o_disconnect)(struct obd_export *exp); /* Initialize/finalize fids infrastructure. */ @@ -1244,7 +1245,7 @@ struct obd_ops { int (*o_fid_alloc)(struct obd_export *exp, struct lu_fid *fid, struct md_op_data *op_data); - /* + /* * Object with @fid is getting deleted, we may want to do something * about this. */ diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 2a7cd5e5c3..045e3ff269 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -832,7 +832,8 @@ static inline int obd_reconnect(const struct lu_env *env, struct obd_export *exp, struct obd_device *obd, struct obd_uuid *cluuid, - struct obd_connect_data *d) + struct obd_connect_data *d, + void *localdata) { int rc; #ifdef LIBCFS_DEBUG @@ -846,7 +847,7 @@ static inline int obd_reconnect(const struct lu_env *env, OBD_CHECK_DT_OP(obd, reconnect, 0); OBD_COUNTER_INCREMENT(obd, reconnect); - rc = OBP(obd, reconnect)(env, exp, obd, cluuid, d); + rc = OBP(obd, reconnect)(env, exp, obd, cluuid, d, localdata); /* check that only subset is granted */ LASSERT(ergo(d != NULL, (d->ocd_connect_flags & ocf) == d->ocd_connect_flags)); diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index 4d8a62d14e..44b3159718 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -588,7 +588,7 @@ void target_client_add_cb(struct obd_device *obd, __u64 transno, void *cb_data, spin_unlock(&exp->exp_lock); } EXPORT_SYMBOL(target_client_add_cb); -static void +static void target_start_and_reset_recovery_timer(struct obd_device *obd, struct ptlrpc_request *req, int new_client); @@ -798,7 +798,7 @@ int target_handle_connect(struct ptlrpc_request *req) /* If this is the first time a client connects, reset the recovery timer */ if (rc == 0) - target_start_and_reset_recovery_timer(target, req, + target_start_and_reset_recovery_timer(target, req, !export); } @@ -837,7 +837,7 @@ dont_check_exports: } } else { rc = obd_reconnect(req->rq_svc_thread->t_env, - export, target, &cluuid, data); + export, target, &cluuid, data, client_nid); } if (rc) GOTO(out, rc); @@ -1254,7 +1254,7 @@ void target_cancel_recovery_timer(struct obd_device *obd) CDEBUG(D_HA, "%s: cancel recovery timer\n", obd->obd_name); cfs_timer_disarm(&obd->obd_recovery_timer); } - + /* extend = 1 means require at least "duration" seconds left in the timer, extend = 0 means set the total duration (start_recovery_timer) */ static void reset_recovery_timer(struct obd_device *obd, int duration, @@ -1277,15 +1277,15 @@ static void reset_recovery_timer(struct obd_device *obd, int duration, /* Track the client's largest expected replay time */ obd->obd_recovery_timeout = duration; #ifdef CRAY_XT3 - /* - * If total recovery time already exceed the - * obd_recovery_max_time, then CRAY XT3 will + /* + * If total recovery time already exceed the + * obd_recovery_max_time, then CRAY XT3 will * abort the recovery */ if(obd->obd_recovery_timeout > obd->obd_recovery_max_time) obd->obd_recovery_timeout = obd->obd_recovery_max_time; #endif - obd->obd_recovery_end = obd->obd_recovery_start + + obd->obd_recovery_end = obd->obd_recovery_start + obd->obd_recovery_timeout; if (!cfs_timer_is_armed(&obd->obd_recovery_timer) || cfs_time_before(now, obd->obd_recovery_end)) { @@ -1315,12 +1315,12 @@ static void check_and_start_recovery_timer(struct obd_device *obd) /* Reset the timer with each new client connection */ /* - * This timer is actually reconnect_timer, which is for making sure - * the total recovery window is at least as big as my reconnect + * This timer is actually reconnect_timer, which is for making sure + * the total recovery window is at least as big as my reconnect * attempt timing. So the initial recovery time_out will be set to * OBD_RECOVERY_FACTOR * obd_timeout. If the timeout coming * from client is bigger than this, then the recovery time_out will - * be extend to make sure the client could be reconnected, in the + * be extend to make sure the client could be reconnected, in the * process, the timeout from the new client should be ignored. */ @@ -1329,7 +1329,7 @@ target_start_and_reset_recovery_timer(struct obd_device *obd, struct ptlrpc_request *req, int new_client) { - int req_timeout = OBD_RECOVERY_FACTOR * + int req_timeout = OBD_RECOVERY_FACTOR * lustre_msg_get_timeout(req->rq_reqmsg); check_and_start_recovery_timer(obd); @@ -1989,9 +1989,9 @@ int target_pack_pool_reply(struct ptlrpc_request *req) { struct obd_device *obd; ENTRY; - - /* - * Check that we still have all structures alive as this may + + /* + * Check that we still have all structures alive as this may * be some late rpc in shutdown time. */ if (unlikely(!req->rq_export || !req->rq_export->exp_obd || @@ -2001,8 +2001,8 @@ int target_pack_pool_reply(struct ptlrpc_request *req) RETURN(0); } - /* - * OBD is alive here as export is alive, which we checked above. + /* + * OBD is alive here as export is alive, which we checked above. */ obd = req->rq_export->exp_obd; diff --git a/lustre/mdt/Makefile.in b/lustre/mdt/Makefile.in index e06fd34c9a..e88367c4aa 100644 --- a/lustre/mdt/Makefile.in +++ b/lustre/mdt/Makefile.in @@ -1,5 +1,5 @@ MODULES := mdt mdt-objs := mdt_handler.o mdt_lib.o mdt_reint.o mdt_xattr.o mdt_recovery.o -mdt-objs += mdt_open.o mdt_idmap.o mdt_identity.o mdt_capa.o mdt_lproc.o +mdt-objs += mdt_open.o mdt_idmap.o mdt_identity.o mdt_capa.o mdt_lproc.o mdt_fs.o @INCLUDE_RULES@ diff --git a/lustre/mdt/mdt_fs.c b/lustre/mdt/mdt_fs.c new file mode 100644 index 0000000000..5ce28b4f1d --- /dev/null +++ b/lustre/mdt/mdt_fs.c @@ -0,0 +1,78 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + * GPL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Use is subject to license terms. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. + * + * lustre/mdt/mdt_fs.c + * + * Lustre Metadata Server (MDS) filesystem interface code + */ +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif +#define DEBUG_SUBSYSTEM S_MDS + +#include "mdt_internal.h" + +int mdt_export_stats_init(struct obd_device *obd, + struct obd_export *exp, + void *localdata) +{ + lnet_nid_t *client_nid = localdata; + int rc, newnid; + + rc = lprocfs_exp_setup(exp, client_nid, &newnid); + if (rc) { + /* Mask error for already created + * /proc entries */ + if (rc == -EALREADY) + rc = 0; + return rc; + } + + if ((obd->md_stats == NULL) && + (rc = lprocfs_alloc_md_stats(obd, LPROC_MDT_NR))) + return rc; + if (newnid) { + /* Always add in ldlm_stats */ + exp->exp_nid_stats->nid_ldlm_stats = + lprocfs_alloc_stats(LDLM_LAST_OPC - LDLM_FIRST_OPC, 0); + if (exp->exp_nid_stats->nid_ldlm_stats == NULL) + return -ENOMEM; + lprocfs_init_ldlm_stats(exp->exp_nid_stats->nid_ldlm_stats); + rc = lprocfs_register_stats(exp->exp_nid_stats->nid_proc, + "ldlm_stats", + exp->exp_nid_stats->nid_ldlm_stats); + } + return rc; +} diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 6fc8ae9d0c..8481e32a69 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -3830,6 +3830,9 @@ static void mdt_fini(const struct lu_env *env, struct mdt_device *m) mdt_seq_fini_cli(m); mdt_fld_fini(env, m); mdt_procfs_fini(m); + lprocfs_remove_proc_entry("clear", obd->obd_proc_exports_entry); + lprocfs_free_per_client_stats(obd); + lprocfs_free_obd_stats(obd); ptlrpc_lprocfs_unregister_obd(d->ld_obd); lprocfs_obd_cleanup(d->ld_obd); @@ -3981,6 +3984,13 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, GOTO(err_fini_proc, rc); } + obd->obd_proc_exports_entry = proc_mkdir("exports", + obd->obd_proc_entry); + if (obd->obd_proc_exports_entry) + lprocfs_add_simple(obd->obd_proc_exports_entry, + "clear", lprocfs_nid_stats_clear_read, + lprocfs_nid_stats_clear_write, obd, NULL); + /* set server index */ LASSERT(num); lu_site2md(s)->ms_node_id = simple_strtol(num, NULL, 10); @@ -4098,6 +4108,8 @@ err_fini_stack: mdt_stack_fini(env, m, md2lu_dev(m->mdt_child)); err_fini_proc: mdt_procfs_fini(m); + if (obd->obd_proc_exports_entry) + lprocfs_remove_proc_entry("clear", obd->obd_proc_exports_entry); ptlrpc_lprocfs_unregister_obd(obd); lprocfs_obd_cleanup(obd); err_fini_site: @@ -4393,6 +4405,8 @@ static int mdt_obd_connect(const struct lu_env *env, if (rc != 0) { OBD_FREE_PTR(lcd); exp->exp_mdt_data.med_lcd = NULL; + } else { + mdt_export_stats_init(obd, exp, localdata); } } else rc = -ENOMEM; @@ -4409,7 +4423,8 @@ static int mdt_obd_connect(const struct lu_env *env, static int mdt_obd_reconnect(const struct lu_env *env, struct obd_export *exp, struct obd_device *obd, struct obd_uuid *cluuid, - struct obd_connect_data *data) + struct obd_connect_data *data, + void *localdata) { struct mdt_thread_info *info; struct mdt_device *mdt; @@ -4448,6 +4463,8 @@ static int mdt_obd_reconnect(const struct lu_env *env, spin_unlock(&exp->exp_lock); rc = mdt_connect_internal(exp, mdt_dev(obd->obd_lu_dev), data); + if (rc == 0) + mdt_export_stats_init(obd, exp, localdata); RETURN(rc); } diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index d68fe3c5e3..5457a0535a 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -389,10 +389,10 @@ struct mdt_txn_info { extern struct lu_context_key mdt_txn_key; static inline void mdt_trans_add_cb(const struct thandle *th, - mdt_cb_t cb_func, void *cb_data) + mdt_cb_t cb_func, void *cb_data) { struct mdt_txn_info *txi; - + txi = lu_context_key_get(&th->th_ctx, &mdt_txn_key); LASSERT(txi->txi_cb_count < ARRAY_SIZE(txi->txi_cb)); @@ -541,6 +541,10 @@ int mdt_client_add(const struct lu_env *env, int mdt_client_new(const struct lu_env *env, struct mdt_device *mdt); +int mdt_export_stats_init(struct obd_device *obd, + struct obd_export *exp, + void *client_nid); + int mdt_pin(struct mdt_thread_info* info); int mdt_lock_new_child(struct mdt_thread_info *info, diff --git a/lustre/mdt/mdt_recovery.c b/lustre/mdt/mdt_recovery.c index 987a740a61..333d13ca1d 100644 --- a/lustre/mdt/mdt_recovery.c +++ b/lustre/mdt/mdt_recovery.c @@ -682,6 +682,7 @@ int mdt_client_add(const struct lu_env *env, init_mutex(&med->med_lcd_lock); LASSERTF(med->med_lr_off > 0, "med_lr_off = %llu\n", med->med_lr_off); + mdt_export_stats_init(obd, mti->mti_exp, NULL); RETURN(rc); } diff --git a/lustre/mgs/mgs_fs.c b/lustre/mgs/mgs_fs.c index 4b87e84afd..2812f84c7f 100644 --- a/lustre/mgs/mgs_fs.c +++ b/lustre/mgs/mgs_fs.c @@ -73,9 +73,9 @@ static int mgs_export_stats_init(struct obd_device *obd, struct obd_export *exp, return rc; } - if (!obd->md_stats && - lprocfs_alloc_md_stats(obd, LPROC_MGS_LAST)) - return rc; + if ((obd->md_stats == NULL) && + (rc = lprocfs_alloc_md_stats(obd, LPROC_MGS_LAST))) + return rc; if (newnid) { /* Always add in ldlm_stats */ exp->exp_nid_stats->nid_ldlm_stats = diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c index aca7b6dc63..f2a1a535e0 100644 --- a/lustre/mgs/mgs_handler.c +++ b/lustre/mgs/mgs_handler.c @@ -103,7 +103,8 @@ static int mgs_connect(const struct lu_env *env, static int mgs_reconnect(const struct lu_env *env, struct obd_export *exp, struct obd_device *obd, - struct obd_uuid *cluuid, struct obd_connect_data *data) + struct obd_uuid *cluuid, struct obd_connect_data *data, + void *localdata) { ENTRY; diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c index 9c346bfe3e..0e10ec8a0b 100644 --- a/lustre/obdfilter/filter.c +++ b/lustre/obdfilter/filter.c @@ -213,8 +213,13 @@ static int filter_export_stats_init(struct obd_device *obd, RETURN(0); rc = lprocfs_exp_setup(exp, client_nid, &newnid); - if (rc) + if (rc) { + /* Mask error for already created + * /proc entries */ + if (rc == -EALREADY) + rc = 0; RETURN(rc); + } if (newnid) { struct nid_stat *tmp = exp->exp_nid_stats; @@ -239,6 +244,17 @@ static int filter_export_stats_init(struct obd_device *obd, tmp->nid_stats); if (rc) RETURN(rc); + /* Always add in ldlm_stats */ + tmp->nid_ldlm_stats = lprocfs_alloc_stats(LDLM_LAST_OPC - + LDLM_FIRST_OPC, 0); + if (tmp->nid_ldlm_stats == NULL) + return -ENOMEM; + + lprocfs_init_ldlm_stats(tmp->nid_ldlm_stats); + rc = lprocfs_register_stats(tmp->nid_proc, "ldlm_stats", + tmp->nid_ldlm_stats); + if (rc) + RETURN(rc); } RETURN(0); @@ -1417,7 +1433,7 @@ struct dentry *filter_parent(struct obd_device *obd, obd_gr group, obd_id objid) struct filter_subdirs *subdirs; LASSERT(group < filter->fo_group_count); /* FIXME: object groups */ - if ((group > 0 && group < FILTER_GROUP_MDS0) || + if ((group > 0 && group < FILTER_GROUP_MDS0) || filter->fo_subdir_count == 0) return filter->fo_dentry_O_groups[group]; @@ -2438,7 +2454,7 @@ static int filter_llog_connect(struct obd_export *exp, int rc; ENTRY; - CDEBUG(D_OTHER, "%s: LLog connect for: "LPX64"/"LPX64":%x\n", + CDEBUG(D_OTHER, "%s: LLog connect for: "LPX64"/"LPX64":%x\n", obd->obd_name, body->lgdc_logid.lgl_oid, body->lgdc_logid.lgl_ogr, body->lgdc_logid.lgl_ogen); @@ -2455,7 +2471,7 @@ static int filter_llog_connect(struct obd_export *exp, body->lgdc_ctxt_idx); CWARN("%s: Recovery from log "LPX64"/"LPX64":%x\n", - obd->obd_name, body->lgdc_logid.lgl_oid, + obd->obd_name, body->lgdc_logid.lgl_oid, body->lgdc_logid.lgl_ogr, body->lgdc_logid.lgl_ogen); rc = llog_connect(ctxt, &body->lgdc_logid, @@ -2684,7 +2700,8 @@ static int filter_connect_internal(struct obd_export *exp, static int filter_reconnect(const struct lu_env *env, struct obd_export *exp, struct obd_device *obd, struct obd_uuid *cluuid, - struct obd_connect_data *data) + struct obd_connect_data *data, + void *localdata) { int rc; ENTRY; @@ -2693,6 +2710,8 @@ static int filter_reconnect(const struct lu_env *env, RETURN(-EINVAL); rc = filter_connect_internal(exp, data); + if (rc == 0) + filter_export_stats_init(obd, exp, localdata); RETURN(rc); } @@ -3713,7 +3732,7 @@ static int filter_precreate(struct obd_device *obd, struct obdo *oa, cleanup_phase = 3; CDEBUG(D_INODE, "%s: filter_precreate(od->o_gr="LPU64 - ",od->o_id="LPU64")\n", obd->obd_name, group, + ",od->o_id="LPU64")\n", obd->obd_name, group, next_id); /* We mark object SUID+SGID to flag it for accepting UID+GID diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index 8ce49e27bb..a00185f085 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -3867,7 +3867,7 @@ static int osc_llog_init(struct obd_device *obd, struct obd_llog_group *olg, rc = llog_setup(obd, &obd->obd_olg, LLOG_SIZE_REPL_CTXT, tgt, count, NULL, &osc_size_repl_logops); if (rc) { - struct llog_ctxt *ctxt = + struct llog_ctxt *ctxt = llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT); if (ctxt) llog_cleanup(ctxt); @@ -3906,7 +3906,8 @@ static int osc_llog_finish(struct obd_device *obd, int count) static int osc_reconnect(const struct lu_env *env, struct obd_export *exp, struct obd_device *obd, struct obd_uuid *cluuid, - struct obd_connect_data *data) + struct obd_connect_data *data, + void *localdata) { struct client_obd *cli = &obd->u.cli; diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c index cd1d56bd5b..b575270f6a 100644 --- a/lustre/ptlrpc/import.c +++ b/lustre/ptlrpc/import.c @@ -554,7 +554,7 @@ int ptlrpc_connect_import(struct obd_import *imp, char *new_uuid) imp->imp_msghdr_flags &= ~MSGHDR_AT_SUPPORT; rc = obd_reconnect(NULL, imp->imp_obd->obd_self_export, obd, - &obd->obd_uuid, &imp->imp_connect_data); + &obd->obd_uuid, &imp->imp_connect_data, NULL); if (rc) GOTO(out, rc); -- GitLab