diff --git a/libcfs/libcfs/darwin/darwin-tracefile.c b/libcfs/libcfs/darwin/darwin-tracefile.c
index 8cf3c8042a15573df784da3dfd31f4f5c56b2856..00520948894ff155b620b163a7e2711f62d99bc1 100644
--- a/libcfs/libcfs/darwin/darwin-tracefile.c
+++ b/libcfs/libcfs/darwin/darwin-tracefile.c
@@ -218,9 +218,3 @@ int trace_max_debug_mb(void)
{
return max_permit_mb;
}
-
-void
-trace_call_on_all_cpus(void (*fn)(void *arg), void *arg)
-{
-#error "tbd"
-}
diff --git a/libcfs/libcfs/linux/linux-tracefile.c b/libcfs/libcfs/linux/linux-tracefile.c
index 6111060c3ce36b5bea0b9730351ea803807dcb72..52986c23688f28b3620c680408373572deec32e7 100644
--- a/libcfs/libcfs/linux/linux-tracefile.c
+++ b/libcfs/libcfs/linux/linux-tracefile.c
@@ -84,6 +84,7 @@ int tracefile_init_arch()
/* arch related info initialized */
tcd_for_each(tcd, i, j) {
+ spin_lock_init(&tcd->tcd_lock);
tcd->tcd_pages_factor = pages_factor[i];
tcd->tcd_type = i;
tcd->tcd_cpu = j;
@@ -174,39 +175,44 @@ trace_put_console_buffer(char *buffer)
struct trace_cpu_data *
trace_get_tcd(void)
{
+ struct trace_cpu_data *tcd;
int cpu;
cpu = get_cpu();
if (in_irq())
- return &(*trace_data[TCD_TYPE_IRQ])[cpu].tcd;
+ tcd = &(*trace_data[TCD_TYPE_IRQ])[cpu].tcd;
else if (in_softirq())
- return &(*trace_data[TCD_TYPE_SOFTIRQ])[cpu].tcd;
- return &(*trace_data[TCD_TYPE_PROC])[cpu].tcd;
+ tcd = &(*trace_data[TCD_TYPE_SOFTIRQ])[cpu].tcd;
+ else
+ tcd = &(*trace_data[TCD_TYPE_PROC])[cpu].tcd;
+
+ trace_lock_tcd(tcd);
+
+ return tcd;
}
void
trace_put_tcd (struct trace_cpu_data *tcd)
{
+ trace_unlock_tcd(tcd);
+
put_cpu();
}
int trace_lock_tcd(struct trace_cpu_data *tcd)
{
__LASSERT(tcd->tcd_type < TCD_TYPE_MAX);
- if (tcd->tcd_type == TCD_TYPE_IRQ)
- local_irq_disable();
- else if (tcd->tcd_type == TCD_TYPE_SOFTIRQ)
- local_bh_disable();
+
+ spin_lock_irqsave(&tcd->tcd_lock, tcd->tcd_lock_flags);
+
return 1;
}
void trace_unlock_tcd(struct trace_cpu_data *tcd)
{
__LASSERT(tcd->tcd_type < TCD_TYPE_MAX);
- if (tcd->tcd_type == TCD_TYPE_IRQ)
- local_irq_enable();
- else if (tcd->tcd_type == TCD_TYPE_SOFTIRQ)
- local_bh_enable();
+
+ spin_unlock_irqrestore(&tcd->tcd_lock, tcd->tcd_lock_flags);
}
int tcd_owns_tage(struct trace_cpu_data *tcd, struct trace_page *tage)
@@ -278,26 +284,3 @@ int trace_max_debug_mb(void)
return MAX(512, (total_mb * 80)/100);
}
-
-void
-trace_call_on_all_cpus(void (*fn)(void *arg), void *arg)
-{
- cpumask_t cpus_allowed = current->cpus_allowed;
- /* use cpus_allowed to quiet 2.4 UP kernel warning only */
- cpumask_t m = cpus_allowed;
- int cpu;
-
- /* Run the given routine on every CPU in thread context */
- for (cpu = 0; cpu < num_possible_cpus(); cpu++) {
- if (!cpu_online(cpu))
- continue;
-
- cpus_clear(m);
- cpu_set(cpu, m);
- set_cpus_allowed(current, m);
-
- fn(arg);
-
- set_cpus_allowed(current, cpus_allowed);
- }
-}
diff --git a/libcfs/libcfs/tracefile.c b/libcfs/libcfs/tracefile.c
index eaebd77f4e2bdc946b6caf7da6e03778657f1dfe..e694fbe5e0fc5977f9f5af9c981d929e0c8d7d9a 100644
--- a/libcfs/libcfs/tracefile.c
+++ b/libcfs/libcfs/tracefile.c
@@ -499,19 +499,21 @@ panic_collect_pages(struct page_collection *pc)
}
}
-static void collect_pages_on_cpu(void *info)
+static void collect_pages_on_all_cpus(struct page_collection *pc)
{
struct trace_cpu_data *tcd;
- struct page_collection *pc = info;
- int i;
+ int i, cpu;
spin_lock(&pc->pc_lock);
- tcd_for_each_type_lock(tcd, i) {
- list_splice_init(&tcd->tcd_pages, &pc->pc_pages);
- tcd->tcd_cur_pages = 0;
- if (pc->pc_want_daemon_pages) {
- list_splice_init(&tcd->tcd_daemon_pages, &pc->pc_pages);
- tcd->tcd_cur_daemon_pages = 0;
+ for_each_possible_cpu(cpu) {
+ tcd_for_each_type_lock(tcd, i, cpu) {
+ list_splice_init(&tcd->tcd_pages, &pc->pc_pages);
+ tcd->tcd_cur_pages = 0;
+ if (pc->pc_want_daemon_pages) {
+ list_splice_init(&tcd->tcd_daemon_pages,
+ &pc->pc_pages);
+ tcd->tcd_cur_daemon_pages = 0;
+ }
}
}
spin_unlock(&pc->pc_lock);
@@ -524,32 +526,35 @@ static void collect_pages(struct page_collection *pc)
if (libcfs_panic_in_progress)
panic_collect_pages(pc);
else
- trace_call_on_all_cpus(collect_pages_on_cpu, pc);
+ collect_pages_on_all_cpus(pc);
}
-static void put_pages_back_on_cpu(void *info)
+static void put_pages_back_on_all_cpus(struct page_collection *pc)
{
- struct page_collection *pc = info;
struct trace_cpu_data *tcd;
struct list_head *cur_head;
struct trace_page *tage;
struct trace_page *tmp;
- int i;
+ int i, cpu;
spin_lock(&pc->pc_lock);
- tcd_for_each_type_lock(tcd, i) {
- cur_head = tcd->tcd_pages.next;
+ for_each_possible_cpu(cpu) {
+ tcd_for_each_type_lock(tcd, i, cpu) {
+ cur_head = tcd->tcd_pages.next;
- cfs_list_for_each_entry_safe_typed(tage, tmp, &pc->pc_pages,
- struct trace_page, linkage) {
+ cfs_list_for_each_entry_safe_typed(tage, tmp,
+ &pc->pc_pages,
+ struct trace_page,
+ linkage) {
- __LASSERT_TAGE_INVARIANT(tage);
+ __LASSERT_TAGE_INVARIANT(tage);
- if (tage->cpu != smp_processor_id() || tage->type != i)
- continue;
+ if (tage->cpu != cpu || tage->type != i)
+ continue;
- tage_to_tail(tage, cur_head);
- tcd->tcd_cur_pages++;
+ tage_to_tail(tage, cur_head);
+ tcd->tcd_cur_pages++;
+ }
}
}
spin_unlock(&pc->pc_lock);
@@ -558,7 +563,7 @@ static void put_pages_back_on_cpu(void *info)
static void put_pages_back(struct page_collection *pc)
{
if (!libcfs_panic_in_progress)
- trace_call_on_all_cpus(put_pages_back_on_cpu, pc);
+ put_pages_back_on_all_cpus(pc);
}
/* Add pages to a per-cpu debug daemon ringbuffer. This buffer makes sure that
@@ -577,8 +582,7 @@ static void put_pages_on_tcd_daemon_list(struct page_collection *pc,
__LASSERT_TAGE_INVARIANT(tage);
- if (tage->cpu != smp_processor_id() ||
- tage->type != tcd->tcd_type)
+ if (tage->cpu != tcd->tcd_cpu || tage->type != tcd->tcd_type)
continue;
tage_to_tail(tage, &tcd->tcd_daemon_pages);
@@ -600,18 +604,15 @@ static void put_pages_on_tcd_daemon_list(struct page_collection *pc,
spin_unlock(&pc->pc_lock);
}
-static void put_pages_on_daemon_list_on_cpu(void *info)
+static void put_pages_on_daemon_list(struct page_collection *pc)
{
struct trace_cpu_data *tcd;
- int i;
-
- tcd_for_each_type_lock(tcd, i)
- put_pages_on_tcd_daemon_list(info, tcd);
-}
+ int i, cpu;
-static void put_pages_on_daemon_list(struct page_collection *pc)
-{
- trace_call_on_all_cpus(put_pages_on_daemon_list_on_cpu, pc);
+ for_each_possible_cpu(cpu) {
+ tcd_for_each_type_lock(tcd, i, cpu)
+ put_pages_on_tcd_daemon_list(pc, tcd);
+ }
}
void trace_debug_print(void)
@@ -1125,24 +1126,29 @@ int tracefile_init(int max_pages)
return 0;
}
-static void trace_cleanup_on_cpu(void *info)
+static void trace_cleanup_on_all_cpus(void)
{
struct trace_cpu_data *tcd;
struct trace_page *tage;
struct trace_page *tmp;
- int i;
+ int i, cpu;
- tcd_for_each_type_lock(tcd, i) {
- tcd->tcd_shutting_down = 1;
+ for_each_possible_cpu(cpu) {
+ tcd_for_each_type_lock(tcd, i, cpu) {
+ tcd->tcd_shutting_down = 1;
- cfs_list_for_each_entry_safe_typed(tage, tmp, &tcd->tcd_pages,
- struct trace_page, linkage) {
- __LASSERT_TAGE_INVARIANT(tage);
+ cfs_list_for_each_entry_safe_typed(tage, tmp,
+ &tcd->tcd_pages,
+ struct trace_page,
+ linkage) {
+ __LASSERT_TAGE_INVARIANT(tage);
- list_del(&tage->linkage);
- tage_free(tage);
+ list_del(&tage->linkage);
+ tage_free(tage);
+ }
+
+ tcd->tcd_cur_pages = 0;
}
- tcd->tcd_cur_pages = 0;
}
}
@@ -1153,7 +1159,7 @@ static void trace_cleanup(void)
CFS_INIT_LIST_HEAD(&pc.pc_pages);
spin_lock_init(&pc.pc_lock);
- trace_call_on_all_cpus(trace_cleanup_on_cpu, &pc);
+ trace_cleanup_on_all_cpus();
tracefile_fini_arch();
}
diff --git a/libcfs/libcfs/tracefile.h b/libcfs/libcfs/tracefile.h
index 7faf8039cdc5fe7c22b5765b6294caf2c7d33349..a857fb541bc40b8c6358200dec1b3d394f5f7f67 100644
--- a/libcfs/libcfs/tracefile.h
+++ b/libcfs/libcfs/tracefile.h
@@ -104,6 +104,16 @@ extern int trace_max_debug_mb(void);
union trace_data_union {
struct trace_cpu_data {
+ /*
+ * Even though this structure is meant to be per-CPU, locking
+ * is needed because in some places the data may be accessed
+ * from other CPUs. This lock is directly used in trace_get_tcd
+ * and trace_put_tcd, which are called in libcfs_debug_vmsg2 and
+ * tcd_for_each_type_lock
+ */
+ spinlock_t tcd_lock;
+ unsigned long tcd_lock_flags;
+
/*
* pages with trace records not yet processed by tracefiled.
*/
@@ -176,9 +186,9 @@ extern union trace_data_union (*trace_data[TCD_MAX_TYPES])[NR_CPUS];
for (j = 0, ((tcd) = &(*trace_data[i])[j].tcd); \
j < num_possible_cpus(); j++, (tcd) = &(*trace_data[i])[j].tcd)
-#define tcd_for_each_type_lock(tcd, i) \
+#define tcd_for_each_type_lock(tcd, i, cpu) \
for (i = 0; trace_data[i] && \
- (tcd = &(*trace_data[i])[smp_processor_id()].tcd) && \
+ (tcd = &(*trace_data[i])[cpu].tcd) && \
trace_lock_tcd(tcd); trace_unlock_tcd(tcd), i++)
/* XXX nikita: this declaration is internal to tracefile.c and should probably
@@ -253,8 +263,6 @@ extern void trace_unlock_tcd(struct trace_cpu_data *tcd);
extern char *trace_get_console_buffer(void);
extern void trace_put_console_buffer(char *buffer);
-extern void trace_call_on_all_cpus(void (*fn)(void *arg), void *arg);
-
int trace_refill_stock(struct trace_cpu_data *tcd, int gfp,
struct list_head *stock);
diff --git a/libcfs/libcfs/winnt/winnt-tracefile.c b/libcfs/libcfs/winnt/winnt-tracefile.c
index 6b065d735acb6fc880889857b0a418f38cc8f004..51f70a782ada966731645e335dbe42eeedd26089 100644
--- a/libcfs/libcfs/winnt/winnt-tracefile.c
+++ b/libcfs/libcfs/winnt/winnt-tracefile.c
@@ -259,18 +259,3 @@ int trace_max_debug_mb(void)
return MAX(512, (total_mb * 80)/100);
}
-
-void
-trace_call_on_all_cpus(void (*fn)(void *_arg), void *arg)
-{
- int cpu;
- KAFFINITY mask = cfs_query_thread_affinity();
-
- for (cpu = 0; cpu < num_possible_cpus(); cpu++) {
- if (cfs_tie_thread_to_cpu(cpu)) {
- ASSERT((int)KeGetCurrentProcessorNumber() == cpu);
- fn(arg);
- cfs_set_thread_affinity(mask);
- }
- }
-}