From f674682aae52ed569591e3dccd3b8483a6715457 Mon Sep 17 00:00:00 2001
From: shadow <shadow>
Date: Wed, 11 Jun 2008 06:45:21 +0000
Subject: [PATCH] timeout and assert in invalidate import. b=15716 i=rread
i=johann
---
lustre/ChangeLog | 7 ++
lustre/include/linux/obd_class.h | 4 -
lustre/include/obd_class.h | 4 +
lustre/obdclass/class_obd.c | 1 +
lustre/obdclass/genops.c | 126 ++++++++++++++++++++++++++++---
lustre/obdclass/obd_config.c | 4 +-
lustre/ptlrpc/import.c | 1 +
lustre/ptlrpc/ptlrpcd.c | 13 ----
8 files changed, 132 insertions(+), 28 deletions(-)
diff --git a/lustre/ChangeLog b/lustre/ChangeLog
index 68c8f82135..3c6e6f8852 100644
--- a/lustre/ChangeLog
+++ b/lustre/ChangeLog
@@ -24,6 +24,13 @@ tbd Sun Microsystems, Inc.
'tunefs.lustre --param="mdt.quota_type=ug1" $MDTDEV'.
For more information, please refer to bugzilla 13904.
+Severity : minor
+Bugzilla : 15716
+Description: timeout with invalidate import.
+Details : ptlrpcd_check call obd_zombie_impexp_cull and wait request which should be
+ handled by ptlrpcd. This produce long age waiting and -ETIMEOUT
+ ptlrpc_invalidate_import and as result LASSERT.
+
Severity : enhancement
Bugzilla : 15741
Description: Update to RHEL5 kernel-2.6.18-53.1.19.el5.
diff --git a/lustre/include/linux/obd_class.h b/lustre/include/linux/obd_class.h
index 75a77cd333..dcb5ba12eb 100644
--- a/lustre/include/linux/obd_class.h
+++ b/lustre/include/linux/obd_class.h
@@ -38,10 +38,6 @@
#include <linux/timer.h>
#endif
-void obd_zombie_impexp_init(void);
-void obd_zombie_impexp_cull(void);
-extern void (*obd_zombie_impexp_notify)(void);
-
/* obdo.c */
#ifdef __KERNEL__
void obdo_from_iattr(struct obdo *oa, struct iattr *attr, unsigned ia_valid);
diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h
index 01dad5a126..a6d1483fce 100644
--- a/lustre/include/obd_class.h
+++ b/lustre/include/obd_class.h
@@ -84,6 +84,10 @@ char *obd_export_nid2str(struct obd_export *exp);
int obd_export_evict_by_nid(struct obd_device *obd, char *nid);
int obd_export_evict_by_uuid(struct obd_device *obd, char *uuid);
+int obd_zombie_impexp_init(void);
+void obd_zombie_impexp_stop(void);
+void obd_zombie_impexp_cull(void);
+
/* obd_config.c */
int class_process_config(struct lustre_cfg *lcfg);
int class_process_proc_param(char *prefix, struct lprocfs_vars *lvars,
diff --git a/lustre/obdclass/class_obd.c b/lustre/obdclass/class_obd.c
index c97ca561ad..ef4cc0ecdb 100644
--- a/lustre/obdclass/class_obd.c
+++ b/lustre/obdclass/class_obd.c
@@ -610,6 +610,7 @@ static void cleanup_obdclass(void)
class_handle_cleanup();
class_exit_uuidlist();
+ obd_zombie_impexp_stop();
memory_leaked = obd_memory_sum();
pages_leaked = obd_pages_sum();
diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c
index 57eaa22990..102f06d514 100644
--- a/lustre/obdclass/genops.c
+++ b/lustre/obdclass/genops.c
@@ -45,9 +45,7 @@ cfs_mem_cache_t *import_cachep;
struct list_head obd_zombie_imports;
struct list_head obd_zombie_exports;
spinlock_t obd_zombie_impexp_lock;
-void (*obd_zombie_impexp_notify)(void) = NULL;
-EXPORT_SYMBOL(obd_zombie_impexp_notify);
-
+static void obd_zombie_impexp_notify(void);
int (*ptlrpc_put_connection_superhack)(struct ptlrpc_connection *c);
@@ -1282,11 +1280,11 @@ int obd_export_evict_by_uuid(struct obd_device *obd, char *uuid)
}
EXPORT_SYMBOL(obd_export_evict_by_uuid);
-void obd_zombie_impexp_cull(void)
+void obd_zombie_impexp_cull(void)
{
struct obd_import *import;
struct obd_export *export;
-
+
do {
spin_lock (&obd_zombie_impexp_lock);
@@ -1297,7 +1295,7 @@ void obd_zombie_impexp_cull(void)
imp_zombie_chain);
list_del(&import->imp_zombie_chain);
}
-
+
export = NULL;
if (!list_empty(&obd_zombie_exports)) {
export = list_entry(obd_zombie_exports.next,
@@ -1307,7 +1305,7 @@ void obd_zombie_impexp_cull(void)
}
spin_unlock(&obd_zombie_impexp_lock);
-
+
if (import != NULL)
class_import_destroy(import);
@@ -1316,11 +1314,121 @@ void obd_zombie_impexp_cull(void)
} while (import != NULL || export != NULL);
}
-EXPORT_SYMBOL(obd_zombie_impexp_cull);
-void obd_zombie_impexp_init(void)
+static struct completion obd_zombie_start;
+static struct completion obd_zombie_stop;
+static unsigned long obd_zombie_flags;
+static cfs_waitq_t obd_zombie_waitq;
+
+enum {
+ OBD_ZOMBIE_STOP = 1
+};
+
+int obd_zombi_impexp_check(void *arg)
{
+ int rc;
+
+ spin_lock(&obd_zombie_impexp_lock);
+ rc = list_empty(&obd_zombie_imports) &&
+ list_empty(&obd_zombie_exports) &&
+ !test_bit(OBD_ZOMBIE_STOP, &obd_zombie_flags);
+
+ spin_unlock(&obd_zombie_impexp_lock);
+
+ RETURN(rc);
+}
+
+static void obd_zombie_impexp_notify(void)
+{
+ cfs_waitq_signal(&obd_zombie_waitq);
+}
+
+#ifdef __KERNEL__
+
+static int obd_zombie_impexp_thread(void *unused)
+{
+ int rc;
+
+ if ((rc = cfs_daemonize_ctxt("obd_zombid"))) {
+ complete(&obd_zombie_start);
+ RETURN(rc);
+ }
+
+ complete(&obd_zombie_start);
+
+ while(!test_bit(OBD_ZOMBIE_STOP, &obd_zombie_flags)) {
+ struct l_wait_info lwi = { 0 };
+
+ l_wait_event(obd_zombie_waitq, !obd_zombi_impexp_check(NULL), &lwi);
+
+ obd_zombie_impexp_cull();
+ }
+
+ complete(&obd_zombie_stop);
+
+ RETURN(0);
+}
+
+#else /* ! KERNEL */
+
+static atomic_t zombi_recur = ATOMIC_INIT(0);
+static void *obd_zombi_impexp_work_cb;
+static void *obd_zombi_impexp_idle_cb;
+
+int obd_zombi_impexp_kill(void *arg)
+{
+ int rc = 0;
+
+ if (atomic_inc_return(&zombi_recur) == 1) {
+ obd_zombie_impexp_cull();
+ rc = 1;
+ }
+ atomic_dec(&zombi_recur);
+ return rc;
+}
+
+#endif
+
+int obd_zombie_impexp_init(void)
+{
+ int rc;
+
CFS_INIT_LIST_HEAD(&obd_zombie_imports);
CFS_INIT_LIST_HEAD(&obd_zombie_exports);
spin_lock_init(&obd_zombie_impexp_lock);
+ init_completion(&obd_zombie_start);
+ init_completion(&obd_zombie_stop);
+ cfs_waitq_init(&obd_zombie_waitq);
+
+#ifdef __KERNEL__
+ rc = cfs_kernel_thread(obd_zombie_impexp_thread, NULL, 0);
+ if (rc < 0)
+ RETURN(rc);
+
+ wait_for_completion(&obd_zombie_start);
+#else
+
+ obd_zombi_impexp_work_cb =
+ liblustre_register_wait_callback("obd_zombi_impexp_kill",
+ &obd_zombi_impexp_kill, NULL);
+
+ obd_zombi_impexp_idle_cb =
+ liblustre_register_idle_callback("obd_zombi_impexp_check",
+ &obd_zombi_impexp_check, NULL);
+ rc = 0;
+
+#endif
+ RETURN(rc);
+}
+
+void obd_zombie_impexp_stop(void)
+{
+ set_bit(OBD_ZOMBIE_STOP, &obd_zombie_flags);
+ obd_zombie_impexp_notify();
+#ifdef __KERNEL__
+ wait_for_completion(&obd_zombie_stop);
+#else
+ liblustre_deregister_wait_callback(obd_zombi_impexp_work_cb);
+ liblustre_deregister_idle_callback(obd_zombi_impexp_idle_cb);
+#endif
}
diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c
index cf8c393752..924c6fb48f 100644
--- a/lustre/obdclass/obd_config.c
+++ b/lustre/obdclass/obd_config.c
@@ -359,10 +359,10 @@ int class_detach(struct obd_device *obd, struct lustre_cfg *lcfg)
obd->obd_name, obd->obd_uuid.uuid);
class_decref(obd);
-
+
/* not strictly necessary, but cleans up eagerly */
obd_zombie_impexp_cull();
-
+
RETURN(0);
}
diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c
index 8d60653d6e..e65d1457a0 100644
--- a/lustre/ptlrpc/import.c
+++ b/lustre/ptlrpc/import.c
@@ -234,6 +234,7 @@ void ptlrpc_invalidate_import(struct obd_import *imp)
DEBUG_REQ(D_ERROR, req, "still on delayed list");
}
spin_unlock(&imp->imp_lock);
+ LASSERT(atomic_read(&imp->imp_inflight) == 0);
}
out:
diff --git a/lustre/ptlrpc/ptlrpcd.c b/lustre/ptlrpc/ptlrpcd.c
index d0abb2eb85..5b82f570f0 100644
--- a/lustre/ptlrpc/ptlrpcd.c
+++ b/lustre/ptlrpc/ptlrpcd.c
@@ -97,8 +97,6 @@ static int ptlrpcd_check(struct ptlrpcd_ctl *pc)
if (test_bit(LIOD_STOP, &pc->pc_flags))
RETURN(1);
- obd_zombie_impexp_cull();
-
spin_lock(&pc->pc_set->set_new_req_lock);
list_for_each_safe(pos, tmp, &pc->pc_set->set_new_requests) {
req = list_entry(pos, struct ptlrpc_request, rq_set_chain);
@@ -176,13 +174,6 @@ static int ptlrpcd(void *arg)
return 0;
}
-static void ptlrpcd_zombie_impexp_notify(void)
-{
- LASSERT(ptlrpcd_pc.pc_set != NULL); // call before ptlrpcd inited ?
-
- cfs_waitq_signal(&ptlrpcd_pc.pc_set->set_waitq);
-}
-
#else
int ptlrpcd_check_async_rpcs(void *arg)
@@ -233,9 +224,6 @@ static int ptlrpcd_start(char *name, struct ptlrpcd_ctl *pc)
RETURN(-ENOMEM);
#ifdef __KERNEL__
- /* wake ptlrpcd when zombie imports or exports exist */
- obd_zombie_impexp_notify = ptlrpcd_zombie_impexp_notify;
-
rc = cfs_kernel_thread(ptlrpcd, pc, 0);
if (rc < 0) {
ptlrpc_set_destroy(pc->pc_set);
@@ -260,7 +248,6 @@ static void ptlrpcd_stop(struct ptlrpcd_ctl *pc)
set_bit(LIOD_STOP, &pc->pc_flags);
cfs_waitq_signal(&pc->pc_set->set_waitq);
#ifdef __KERNEL__
- obd_zombie_impexp_notify = NULL;
wait_for_completion(&pc->pc_finishing);
#else
liblustre_deregister_wait_callback(pc->pc_wait_callback);
--
GitLab