Commit ed86a590 authored by Mathieu Desnoyers's avatar Mathieu Desnoyers Committed by Ingo Molnar

markers: re-enable fast batch registration

Lai Jiangshan discovered a reentrancy issue with markers and fixed it by
adding synchronize_sched() calls at each registration/unregistraiton.

It works, but it removes the ability to do batch
registration/unregistration and can cause registration of ~100 markers
to take about 30 seconds on a loaded machine (synchronize_sched() is
much slower on such workloads).

This patch implements a version of the fix which won't slow down marker batch
registration/unregistration. It also go back to the original non-synchronized
reg/unreg.
Signed-off-by: default avatarMathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 5b9261d9
...@@ -60,6 +60,9 @@ struct marker_entry { ...@@ -60,6 +60,9 @@ struct marker_entry {
struct marker_probe_closure single; struct marker_probe_closure single;
struct marker_probe_closure *multi; struct marker_probe_closure *multi;
int refcount; /* Number of times armed. 0 if disarmed. */ int refcount; /* Number of times armed. 0 if disarmed. */
struct rcu_head rcu;
void *oldptr;
unsigned char rcu_pending:1;
unsigned char ptype:1; unsigned char ptype:1;
char name[0]; /* Contains name'\0'format'\0' */ char name[0]; /* Contains name'\0'format'\0' */
}; };
...@@ -196,6 +199,16 @@ void marker_probe_cb_noarg(const struct marker *mdata, void *call_private, ...) ...@@ -196,6 +199,16 @@ void marker_probe_cb_noarg(const struct marker *mdata, void *call_private, ...)
} }
EXPORT_SYMBOL_GPL(marker_probe_cb_noarg); EXPORT_SYMBOL_GPL(marker_probe_cb_noarg);
static void free_old_closure(struct rcu_head *head)
{
struct marker_entry *entry = container_of(head,
struct marker_entry, rcu);
kfree(entry->oldptr);
/* Make sure we free the data before setting the pending flag to 0 */
smp_wmb();
entry->rcu_pending = 0;
}
static void debug_print_probes(struct marker_entry *entry) static void debug_print_probes(struct marker_entry *entry)
{ {
int i; int i;
...@@ -404,6 +417,7 @@ static struct marker_entry *add_marker(const char *name, const char *format) ...@@ -404,6 +417,7 @@ static struct marker_entry *add_marker(const char *name, const char *format)
e->multi = NULL; e->multi = NULL;
e->ptype = 0; e->ptype = 0;
e->refcount = 0; e->refcount = 0;
e->rcu_pending = 0;
hlist_add_head(&e->hlist, head); hlist_add_head(&e->hlist, head);
return e; return e;
} }
...@@ -433,6 +447,9 @@ static int remove_marker(const char *name) ...@@ -433,6 +447,9 @@ static int remove_marker(const char *name)
if (e->single.func != __mark_empty_function) if (e->single.func != __mark_empty_function)
return -EBUSY; return -EBUSY;
hlist_del(&e->hlist); hlist_del(&e->hlist);
/* Make sure the call_rcu has been executed */
if (e->rcu_pending)
rcu_barrier_sched();
kfree(e); kfree(e);
return 0; return 0;
} }
...@@ -462,8 +479,12 @@ static int marker_set_format(struct marker_entry **entry, const char *format) ...@@ -462,8 +479,12 @@ static int marker_set_format(struct marker_entry **entry, const char *format)
e->multi = (*entry)->multi; e->multi = (*entry)->multi;
e->ptype = (*entry)->ptype; e->ptype = (*entry)->ptype;
e->refcount = (*entry)->refcount; e->refcount = (*entry)->refcount;
e->rcu_pending = 0;
hlist_add_before(&e->hlist, &(*entry)->hlist); hlist_add_before(&e->hlist, &(*entry)->hlist);
hlist_del(&(*entry)->hlist); hlist_del(&(*entry)->hlist);
/* Make sure the call_rcu has been executed */
if ((*entry)->rcu_pending)
rcu_barrier_sched();
kfree(*entry); kfree(*entry);
*entry = e; *entry = e;
trace_mark(core_marker_format, "name %s format %s", trace_mark(core_marker_format, "name %s format %s",
...@@ -637,6 +658,12 @@ int marker_probe_register(const char *name, const char *format, ...@@ -637,6 +658,12 @@ int marker_probe_register(const char *name, const char *format,
goto end; goto end;
} }
} }
/*
* If we detect that a call_rcu is pending for this marker,
* make sure it's executed now.
*/
if (entry->rcu_pending)
rcu_barrier_sched();
old = marker_entry_add_probe(entry, probe, probe_private); old = marker_entry_add_probe(entry, probe, probe_private);
if (IS_ERR(old)) { if (IS_ERR(old)) {
ret = PTR_ERR(old); ret = PTR_ERR(old);
...@@ -644,11 +671,16 @@ int marker_probe_register(const char *name, const char *format, ...@@ -644,11 +671,16 @@ int marker_probe_register(const char *name, const char *format,
} }
mutex_unlock(&markers_mutex); mutex_unlock(&markers_mutex);
marker_update_probes(); /* may update entry */ marker_update_probes(); /* may update entry */
synchronize_sched();
kfree(old);
mutex_lock(&markers_mutex); mutex_lock(&markers_mutex);
entry = get_marker(name); entry = get_marker(name);
WARN_ON(!entry); WARN_ON(!entry);
if (entry->rcu_pending)
rcu_barrier_sched();
entry->oldptr = old;
entry->rcu_pending = 1;
/* write rcu_pending before calling the RCU callback */
smp_wmb();
call_rcu_sched(&entry->rcu, free_old_closure);
end: end:
mutex_unlock(&markers_mutex); mutex_unlock(&markers_mutex);
return ret; return ret;
...@@ -678,15 +710,22 @@ int marker_probe_unregister(const char *name, ...@@ -678,15 +710,22 @@ int marker_probe_unregister(const char *name,
entry = get_marker(name); entry = get_marker(name);
if (!entry) if (!entry)
goto end; goto end;
if (entry->rcu_pending)
rcu_barrier_sched();
old = marker_entry_remove_probe(entry, probe, probe_private); old = marker_entry_remove_probe(entry, probe, probe_private);
mutex_unlock(&markers_mutex); mutex_unlock(&markers_mutex);
marker_update_probes(); /* may update entry */ marker_update_probes(); /* may update entry */
synchronize_sched();
kfree(old);
mutex_lock(&markers_mutex); mutex_lock(&markers_mutex);
entry = get_marker(name); entry = get_marker(name);
if (!entry) if (!entry)
goto end; goto end;
if (entry->rcu_pending)
rcu_barrier_sched();
entry->oldptr = old;
entry->rcu_pending = 1;
/* write rcu_pending before calling the RCU callback */
smp_wmb();
call_rcu_sched(&entry->rcu, free_old_closure);
remove_marker(name); /* Ignore busy error message */ remove_marker(name); /* Ignore busy error message */
ret = 0; ret = 0;
end: end:
...@@ -752,14 +791,21 @@ int marker_probe_unregister_private_data(marker_probe_func *probe, ...@@ -752,14 +791,21 @@ int marker_probe_unregister_private_data(marker_probe_func *probe,
ret = -ENOENT; ret = -ENOENT;
goto end; goto end;
} }
if (entry->rcu_pending)
rcu_barrier_sched();
old = marker_entry_remove_probe(entry, NULL, probe_private); old = marker_entry_remove_probe(entry, NULL, probe_private);
mutex_unlock(&markers_mutex); mutex_unlock(&markers_mutex);
marker_update_probes(); /* may update entry */ marker_update_probes(); /* may update entry */
synchronize_sched();
kfree(old);
mutex_lock(&markers_mutex); mutex_lock(&markers_mutex);
entry = get_marker_from_private_data(probe, probe_private); entry = get_marker_from_private_data(probe, probe_private);
WARN_ON(!entry); WARN_ON(!entry);
if (entry->rcu_pending)
rcu_barrier_sched();
entry->oldptr = old;
entry->rcu_pending = 1;
/* write rcu_pending before calling the RCU callback */
smp_wmb();
call_rcu_sched(&entry->rcu, free_old_closure);
remove_marker(entry->name); /* Ignore busy error message */ remove_marker(entry->name); /* Ignore busy error message */
end: end:
mutex_unlock(&markers_mutex); mutex_unlock(&markers_mutex);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment