diff options
| -rw-r--r-- | include/linux/ftrace_event.h | 1 | ||||
| -rw-r--r-- | kernel/trace/trace_events_filter.c | 3 | ||||
| -rw-r--r-- | kernel/trace/trace_events_trigger.c | 1404 |
3 files changed, 1407 insertions, 1 deletions
diff --git a/include/linux/ftrace_event.h b/include/linux/ftrace_event.h index 5961964c0889..87006302819e 100644 --- a/include/linux/ftrace_event.h +++ b/include/linux/ftrace_event.h @@ -353,6 +353,7 @@ enum event_trigger_type { ETT_SNAPSHOT = (1 << 1), ETT_STACKTRACE = (1 << 2), ETT_EVENT_ENABLE = (1 << 3), + ETT_EVENT_HASH = (1 << 4), }; extern void destroy_preds(struct ftrace_event_file *file); diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 60a8e3fc1f4e..cee9b2941e44 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -941,7 +941,8 @@ int filter_assign_type(const char *type) if (strstr(type, "__data_loc") && strstr(type, "char")) return FILTER_DYN_STRING; - if (strchr(type, '[') && strstr(type, "char")) + if (strchr(type, '[') && + (strstr(type, "char") || strstr(type, "u8"))) return FILTER_STATIC_STRING; return FILTER_OTHER; diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index 323846eea79f..210ddd08eb59 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -22,6 +22,9 @@ #include <linux/ctype.h> #include <linux/mutex.h> #include <linux/slab.h> +#include <linux/hash.h> +#include <linux/stacktrace.h> +#include <linux/sort.h> #include "trace.h" @@ -1427,12 +1430,1413 @@ static __init int register_trigger_traceon_traceoff_cmds(void) return ret; } +struct hash_field; + +typedef u64 (*hash_field_fn_t) (struct hash_field *field, void *event); + +struct hash_field { + struct ftrace_event_field *field; + struct ftrace_event_field *aux_field; + hash_field_fn_t fn; + unsigned long flags; +}; + +static u64 hash_field_none(struct hash_field *field, void *event) +{ + return 0; +} + +static u64 hash_field_string(struct hash_field *hash_field, void *event) +{ + char *addr = (char *)(event + hash_field->field->offset); + + return (u64)addr; +} + +static u64 hash_field_diff(struct hash_field *hash_field, void *event) +{ + u64 *m, *s; + + m = (u64 *)(event + hash_field->field->offset); + s = (u64 *)(event + hash_field->aux_field->offset); + + return *m - *s; +} + +#define DEFINE_HASH_FIELD_FN(type) \ +static u64 hash_field_##type(struct hash_field *hash_field, void *event)\ +{ \ + type *addr = (type *)(event + hash_field->field->offset); \ + \ + return (u64)*addr; \ +} + +DEFINE_HASH_FIELD_FN(s64); +DEFINE_HASH_FIELD_FN(u64); +DEFINE_HASH_FIELD_FN(s32); +DEFINE_HASH_FIELD_FN(u32); +DEFINE_HASH_FIELD_FN(s16); +DEFINE_HASH_FIELD_FN(u16); +DEFINE_HASH_FIELD_FN(s8); +DEFINE_HASH_FIELD_FN(u8); + +#define HASH_TRIGGER_BITS 11 +#define COMPOUND_KEY_MAX 8 +#define HASH_VALS_MAX 16 +#define HASH_SORT_KEYS_MAX 2 + +/* Largest event field string currently 32, add 1 = 64 */ +#define HASH_KEY_STRING_MAX 64 + +enum hash_field_flags { + HASH_FIELD_SYM = 1, + HASH_FIELD_HEX = 2, + HASH_FIELD_STACKTRACE = 4, + HASH_FIELD_STRING = 8, + HASH_FIELD_EXECNAME = 16, + HASH_FIELD_SYSCALL = 32, +}; + +enum sort_key_flags { + SORT_KEY_COUNT = 1, +}; + +struct hash_trigger_sort_key { + bool descending; + bool use_hitcount; + bool key_part; + unsigned int idx; +}; + +struct hash_trigger_data { + struct hlist_head *hashtab; + unsigned int hashtab_bits; + char *keys_str; + char *vals_str; + char *sort_keys_str; + struct hash_field *keys[COMPOUND_KEY_MAX]; + unsigned int n_keys; + struct hash_field *vals[HASH_VALS_MAX]; + unsigned int n_vals; + struct ftrace_event_file *event_file; + unsigned long total_hits; + unsigned long total_entries; + struct hash_trigger_sort_key *sort_keys[HASH_SORT_KEYS_MAX]; + struct hash_trigger_sort_key *sort_key_cur; + spinlock_t lock; + unsigned int max_entries; + struct hash_trigger_entry *entries; + unsigned int n_entries; + struct stack_trace *struct_stacktrace_entries; + unsigned int n_struct_stacktrace_entries; + unsigned long *stacktrace_entries; + unsigned int n_stacktrace_entries; + char *hash_key_string_entries; + unsigned int n_hash_key_string_entries; + unsigned long drops; +}; + +enum hash_key_type { + HASH_KEY_TYPE_U64, + HASH_KEY_TYPE_STACKTRACE, + HASH_KEY_TYPE_STRING, +}; + +struct hash_key_part { + enum hash_key_type type; + unsigned long flags; + union { + u64 val_u64; + struct stack_trace *val_stacktrace; + char *val_string; + } var; +}; + +struct hash_trigger_entry { + struct hlist_node node; + struct hash_key_part key_parts[COMPOUND_KEY_MAX]; + u64 sums[HASH_VALS_MAX]; + char comm[TASK_COMM_LEN + 1]; + u64 count; + struct hash_trigger_data* hash_data; +}; + +#define HASH_STACKTRACE_DEPTH 16 +#define HASH_STACKTRACE_SKIP 4 + +static hash_field_fn_t select_value_fn(int field_size, int field_is_signed) +{ + hash_field_fn_t fn = NULL; + + switch (field_size) { + case 8: + if (field_is_signed) + fn = hash_field_s64; + else + fn = hash_field_u64; + break; + case 4: + if (field_is_signed) + fn = hash_field_s32; + else + fn = hash_field_u32; + break; + case 2: + if (field_is_signed) + fn = hash_field_s16; + else + fn = hash_field_u16; + break; + case 1: + if (field_is_signed) + fn = hash_field_s8; + else + fn = hash_field_u8; + break; + } + + return fn; +} + +#define FNV_OFFSET_BASIS (14695981039346656037ULL) +#define FNV_PRIME (1099511628211ULL) + +static u64 hash_fnv_1a(char *key, unsigned int size, unsigned int bits) +{ + u64 hash = FNV_OFFSET_BASIS; + unsigned int i; + + for (i = 0; i < size; i++) { + hash ^= key[i]; + hash *= FNV_PRIME; + } + + return hash >> (64 - bits); +} + +static u64 hash_stacktrace(struct stack_trace *stacktrace, unsigned int bits) +{ + unsigned int size; + + size = stacktrace->nr_entries * sizeof(*stacktrace->entries); + + return hash_fnv_1a((char *)stacktrace->entries, size, bits); +} + +static u64 hash_string(struct hash_field *hash_field, + unsigned int bits, void *rec) +{ + unsigned int size; + char *string; + + size = hash_field->field->size; + string = (char *)hash_field->fn(hash_field, rec); + + return hash_fnv_1a(string, size, bits); +} + +static u64 hash_compound_key(struct hash_trigger_data *hash_data, + unsigned int bits, void *rec) +{ + struct hash_field *hash_field; + u64 key[COMPOUND_KEY_MAX]; + unsigned int i; + + for (i = 0; i < hash_data->n_keys; i++) { + hash_field = hash_data->keys[i]; + key[i] = hash_field->fn(hash_field, rec); + } + + return hash_fnv_1a((char *)key, hash_data->n_keys * sizeof(key[0]), bits); +} + +static u64 hash_key(struct hash_trigger_data *hash_data, void *rec, + struct stack_trace *stacktrace) +{ + /* currently can't have compound key with string or stacktrace */ + struct hash_field *hash_field = hash_data->keys[0]; + unsigned int bits = hash_data->hashtab_bits; + u64 hash_idx = 0; + + if (hash_field->flags & HASH_FIELD_STACKTRACE) + hash_idx = hash_stacktrace(stacktrace, bits); + else if (hash_field->flags & HASH_FIELD_STRING) + hash_idx = hash_string(hash_field, bits, rec); + else if (hash_data->n_keys > 1) + hash_idx = hash_compound_key(hash_data, bits, rec); + else { + u64 hash_val = hash_field->fn(hash_field, rec); + + switch (hash_field->field->size) { + case 8: + hash_idx = hash_64(hash_val, bits); + break; + case 4: + hash_idx = hash_32(hash_val, bits); + break; + default: + WARN_ON_ONCE(1); + break; + } + } + + return hash_idx; +} + +static inline void save_comm(char *comm, struct task_struct *task) { + + if (!task->pid) { + strcpy(comm, "<idle>"); + return; + } + + if (WARN_ON_ONCE(task->pid < 0)) { + strcpy(comm, "<XXX>"); + return; + } + + if (task->pid > PID_MAX_DEFAULT) { + strcpy(comm, "<...>"); + return; + } + + memcpy(comm, task->comm, TASK_COMM_LEN); +} + +static void stacktrace_entry_fill(struct hash_trigger_entry *entry, + unsigned int key, + struct hash_field *hash_field, + struct stack_trace *stacktrace) +{ + struct hash_trigger_data *hash_data = entry->hash_data; + struct stack_trace *stacktrace_copy; + unsigned int size, offset, idx; + + idx = hash_data->n_struct_stacktrace_entries++; + stacktrace_copy = &hash_data->struct_stacktrace_entries[idx]; + *stacktrace_copy = *stacktrace; + + idx = hash_data->n_stacktrace_entries++; + size = sizeof(unsigned long) * HASH_STACKTRACE_DEPTH; + offset = HASH_STACKTRACE_DEPTH * idx; + stacktrace_copy->entries = &hash_data->stacktrace_entries[offset]; + memcpy(stacktrace_copy->entries, stacktrace->entries, size); + + entry->key_parts[key].type = HASH_KEY_TYPE_STACKTRACE; + entry->key_parts[key].flags = hash_field->flags; + entry->key_parts[key].var.val_stacktrace = stacktrace_copy; +} + +static void string_entry_fill(struct hash_trigger_entry *entry, + unsigned int key, + struct hash_field *hash_field, + void *rec) +{ + struct hash_trigger_data *hash_data = entry->hash_data; + unsigned int size = hash_field->field->size + 1; + unsigned int offset; + char *string_copy; + + offset = HASH_KEY_STRING_MAX * hash_data->n_hash_key_string_entries++; + string_copy = &hash_data->hash_key_string_entries[offset]; + + memcpy(string_copy, (char *)hash_field->fn(hash_field, rec), size); + + entry->key_parts[key].type = HASH_KEY_TYPE_STRING; + entry->key_parts[key].flags = hash_field->flags; + entry->key_parts[key].var.val_string = string_copy; +} + +static struct hash_trigger_entry * +hash_trigger_entry_create(struct hash_trigger_data *hash_data, void *rec, + struct stack_trace *stacktrace) +{ + struct hash_trigger_entry *entry = NULL; + struct hash_field *hash_field; + bool save_execname = false; + unsigned int i; + + if (hash_data->n_entries < hash_data->max_entries) { + entry = &hash_data->entries[hash_data->n_entries++]; + if (!entry) + return NULL; + } + + entry->hash_data = hash_data; + + for (i = 0; i < hash_data->n_keys; i++) { + hash_field = hash_data->keys[i]; + + if (hash_field->flags & HASH_FIELD_STACKTRACE) + stacktrace_entry_fill(entry, i, hash_field, stacktrace); + else if (hash_field->flags & HASH_FIELD_STRING) + string_entry_fill(entry, i, hash_field, rec); + else { + u64 hash_val = hash_field->fn(hash_field, rec); + + entry->key_parts[i].type = HASH_KEY_TYPE_U64; + entry->key_parts[i].flags = hash_field->flags; + entry->key_parts[i].var.val_u64 = hash_val; + /* + EXECNAME only applies to common_pid as a + key, And with the assumption that the comm + saved is only for common_pid i.e. current + pid when the event was logged. comm is + saved only when the hash entry is created, + subsequent hits for that hash entry map the + same pid and comm. + */ + if (hash_field->flags & HASH_FIELD_EXECNAME) + save_execname = true; + } + } + + if (save_execname) + save_comm(entry->comm, current); + + return entry; +} + +static void destroy_hashtab(struct hash_trigger_data *hash_data) +{ + struct hlist_head *hashtab = hash_data->hashtab; + + if (!hashtab) + return; + + kfree(hashtab); + + hash_data->hashtab = NULL; +} + +static void destroy_hash_field(struct hash_field *hash_field) +{ + kfree(hash_field); +} + +static struct hash_field * +create_hash_field(struct ftrace_event_field *field, + struct ftrace_event_field *aux_field, + unsigned long flags) +{ + hash_field_fn_t fn = hash_field_none; + struct hash_field *hash_field; + + hash_field = kzalloc(sizeof(struct hash_field), GFP_KERNEL); + if (!hash_field) + return NULL; + + if (flags & HASH_FIELD_STACKTRACE) { + hash_field->flags = flags; + goto out; + } + + if (is_string_field(field)) { + flags |= HASH_FIELD_STRING; + fn = hash_field_string; + } else if (is_function_field(field)) + goto free; + else { + if (aux_field) { + hash_field->aux_field = aux_field; + fn = hash_field_diff; + } else { + fn = select_value_fn(field->size, field->is_signed); + if (!fn) + goto free; + } + } + + hash_field->field = field; + hash_field->fn = fn; + hash_field->flags = flags; + out: + return hash_field; + free: + kfree(hash_field); + hash_field = NULL; + goto out; +} + +static void destroy_hash_fields(struct hash_trigger_data *hash_data) +{ + unsigned int i; + + for (i = 0; i < hash_data->n_keys; i++) { + destroy_hash_field(hash_data->keys[i]); + hash_data->keys[i] = NULL; + } + + for (i = 0; i < hash_data->n_vals; i++) { + destroy_hash_field(hash_data->vals[i]); + hash_data->vals[i] = NULL; + } +} + +static inline struct hash_trigger_sort_key *create_default_sort_key(void) +{ + struct hash_trigger_sort_key *sort_key; + + sort_key = kzalloc(sizeof(*sort_key), GFP_KERNEL); + if (!sort_key) + return NULL; + + sort_key->use_hitcount = true; + + return sort_key; +} + +static inline struct hash_trigger_sort_key * +create_sort_key(char *field_name, struct hash_trigger_data *hash_data) +{ + struct hash_trigger_sort_key *sort_key; + bool key_part = false; + unsigned int j; + + if (!strcmp(field_name, "hitcount")) + return create_default_sort_key(); + + if (strchr(field_name, '-')) { + char *aux_field_name = field_name; + + field_name = strsep(&aux_field_name, "-"); + if (!aux_field_name) + return NULL; + + for (j = 0; j < hash_data->n_vals; j++) + if (!strcmp(field_name, + hash_data->vals[j]->field->name) && + (hash_data->vals[j]->aux_field && + !strcmp(aux_field_name, + hash_data->vals[j]->aux_field->name))) + goto out; + } + + for (j = 0; j < hash_data->n_vals; j++) + if (!strcmp(field_name, hash_data->vals[j]->field->name)) + goto out; + + for (j = 0; j < hash_data->n_keys; j++) { + if (hash_data->keys[j]->flags & HASH_FIELD_STACKTRACE) + continue; + if (hash_data->keys[j]->flags & HASH_FIELD_STRING) + continue; + if (!strcmp(field_name, hash_data->keys[j]->field->name)) { + key_part = true; + goto out; + } + } + + return NULL; + out: + sort_key = kzalloc(sizeof(*sort_key), GFP_KERNEL); + if (!sort_key) + return NULL; + + sort_key->idx = j; + sort_key->key_part = key_part; + + return sort_key; +} + +static int create_sort_keys(struct hash_trigger_data *hash_data) +{ + char *fields_str = hash_data->sort_keys_str; + struct hash_trigger_sort_key *sort_key; + char *field_str, *field_name; + unsigned int i; + int ret = 0; + + if (!fields_str) { + sort_key = create_default_sort_key(); + if (!sort_key) { + ret = -ENOMEM; + goto out; + } + hash_data->sort_keys[0] = sort_key; + goto out; + } + + strsep(&fields_str, "="); + if (!fields_str) { + ret = -EINVAL; + goto free; + } + + for (i = 0; i < HASH_SORT_KEYS_MAX; i++) { + field_str = strsep(&fields_str, ","); + if (!field_str) { + if (i == 0) { + ret = -EINVAL; + goto free; + } else + break; + } + + field_name = strsep(&field_str, "."); + sort_key = create_sort_key(field_name, hash_data); + if (!sort_key) { + ret = -EINVAL; /* or -ENOMEM */ + goto free; + } + if (field_str) { + if (!strcmp(field_str, "descending")) + sort_key->descending = true; + else if (strcmp(field_str, "ascending")) { + ret = -EINVAL; /* not either, err */ + goto free; + } + } + hash_data->sort_keys[i] = sort_key; + } +out: + return ret; +free: + for (i = 0; i < HASH_SORT_KEYS_MAX; i++) { + if (!hash_data->sort_keys[i]) + break; + kfree(hash_data->sort_keys[i]); + hash_data->sort_keys[i] = NULL; + } + goto out; +} + +static int create_key_field(struct hash_trigger_data *hash_data, + unsigned int key, + struct ftrace_event_file *file, + char *field_str) +{ + struct ftrace_event_field *field = NULL; + unsigned long flags = 0; + char *field_name; + int ret = 0; + + if (!strcmp(field_str, "stacktrace")) { + flags |= HASH_FIELD_STACKTRACE; + } else { + field_name = strsep(&field_str, "."); + if (field_str) { + if (!strcmp(field_str, "sym")) + flags |= HASH_FIELD_SYM; + else if (!strcmp(field_str, "hex")) + flags |= HASH_FIELD_HEX; + else if (!strcmp(field_str, "execname") && + !strcmp(field_name, "common_pid")) + flags |= HASH_FIELD_EXECNAME; + else if (!strcmp(field_str, "syscall")) + flags |= HASH_FIELD_SYSCALL; + } + + field = trace_find_event_field(file->event_call, field_name); + if (!field) { + ret = -EINVAL; + goto out; + } + } + + hash_data->keys[key] = create_hash_field(field, NULL, flags); + if (!hash_data->keys[key]) { + ret = -ENOMEM; + goto out; + } + hash_data->n_keys++; + out: + return ret; +} + +static int create_val_field(struct hash_trigger_data *hash_data, + unsigned int val, + struct ftrace_event_file *file, + char *field_str) +{ + struct ftrace_event_field *field = NULL; + unsigned long flags = 0; + char *field_name; + int ret = 0; + + if (!strcmp(field_str, "hitcount")) + return ret; /* There's always a hitcount */ + + field_name = strsep(&field_str, "-"); + if (field_str) { + struct ftrace_event_field *m_field, *s_field; + + m_field = trace_find_event_field(file->event_call, field_name); + if (!m_field || is_string_field(m_field) || + is_function_field(m_field)) { + ret = -EINVAL; + goto out; + } + + s_field = trace_find_event_field(file->event_call, field_str); + if (!s_field || is_string_field(m_field) || + is_function_field(m_field)) { + ret = -EINVAL; + goto out; + } + + hash_data->vals[val] = create_hash_field(m_field, s_field, flags); + if (!hash_data->vals[val]) { + ret = -ENOMEM; + goto out; + } + } else { + field_str = field_name; + field_name = strsep(&field_str, "."); + + if (field_str) { + if (!strcmp(field_str, "sym")) + flags |= HASH_FIELD_SYM; + else if (!strcmp(field_str, "hex")) + flags |= HASH_FIELD_HEX; + } + + field = trace_find_event_field(file->event_call, field_name); + if (!field) { + ret = -EINVAL; + goto out; + } + + hash_data->vals[val] = create_hash_field(field, NULL, flags); + if (!hash_data->vals[val]) { + ret = -ENOMEM; + goto out; + } + } + hash_data->n_vals++; + out: + return ret; +} + +static int create_hash_fields(struct hash_trigger_data *hash_data, + struct ftrace_event_file *file) +{ + char *fields_str, *field_str; + unsigned int i; + int ret = 0; + + fields_str = hash_data->keys_str; + + for (i = 0; i < COMPOUND_KEY_MAX; i++) { + field_str = strsep(&fields_str, ","); + if (!field_str) { + if (i == 0) { + ret = -EINVAL; + goto out; + } else + break; + } + + ret = create_key_field(hash_data, i, file, field_str); + if (ret) + goto out; + } + + fields_str = hash_data->vals_str; + + for (i = 0; i < HASH_VALS_MAX; i++) { + field_str = strsep(&fields_str, ","); + if (!field_str) { + if (i == 0) { + ret = -EINVAL; + goto out; + } else + break; + } + + ret = create_val_field(hash_data, i, file, field_str); + if (ret) + goto out; + } + + ret = create_sort_keys(hash_data); + out: + return ret; +} + +static void destroy_hashdata(struct hash_trigger_data *hash_data) +{ + synchronize_sched(); + + kfree(hash_data->keys_str); + kfree(hash_data->vals_str); + kfree(hash_data->sort_keys_str); + hash_data->keys_str = NULL; + hash_data->vals_str = NULL; + hash_data->sort_keys_str = NULL; + + kfree(hash_data->entries); + hash_data->entries = NULL; + + kfree(hash_data->struct_stacktrace_entries); + hash_data->struct_stacktrace_entries = NULL; + + kfree(hash_data->stacktrace_entries); + hash_data->stacktrace_entries = NULL; + + kfree(hash_data->hash_key_string_entries); + hash_data->hash_key_string_entries = NULL; + + destroy_hash_fields(hash_data); + destroy_hashtab(hash_data); + + kfree(hash_data); +} + +static struct hash_trigger_data *create_hash_data(unsigned int hashtab_bits, + const char *keys, + const char *vals, + const char *sort_keys, + struct ftrace_event_file *file, + int *ret) +{ + unsigned int hashtab_size = (1 << hashtab_bits); + struct hash_trigger_data *hash_data; + unsigned int i, size; + + hash_data = kzalloc(sizeof(*hash_data), GFP_KERNEL); + if (!hash_data) + return NULL; + + /* Let's just say we size for a perfect hash but are not + * perfect. So let's have enough for 2 * the hashtab_size. */ + + /* Also, we'll run out of entries before or at the same time + * we run out of other items like strings or stacks, so we + * only need to pay attention to one counter, for entries. */ + + /* Also, use vmalloc or something for these large blocks. */ + hash_data->max_entries = hashtab_size * 2; + size = sizeof(struct hash_trigger_entry) * hash_data->max_entries; + hash_data->entries = kzalloc(size, GFP_KERNEL); + if (!hash_data->entries) + goto free; + + size = sizeof(struct stack_trace) * hash_data->max_entries; + hash_data->struct_stacktrace_entries = kzalloc(size, GFP_KERNEL); + if (!hash_data->struct_stacktrace_entries) + goto free; + + size = sizeof(unsigned long) * HASH_STACKTRACE_DEPTH * hash_data->max_entries; + hash_data->stacktrace_entries = kzalloc(size, GFP_KERNEL); + if (!hash_data->stacktrace_entries) + goto free; + + size = sizeof(char) * HASH_KEY_STRING_MAX * hash_data->max_entries; + hash_data->hash_key_string_entries = kzalloc(size, GFP_KERNEL); + if (!hash_data->hash_key_string_entries) + goto free; + + hash_data->keys_str = kstrdup(keys, GFP_KERNEL); + hash_data->vals_str = kstrdup(vals, GFP_KERNEL); + if (sort_keys) + hash_data->sort_keys_str = kstrdup(sort_keys, GFP_KERNEL); + + *ret = create_hash_fields(hash_data, file); + if (*ret < 0) + goto free; + + hash_data->hashtab = kzalloc(hashtab_size * sizeof(struct hlist_head), + GFP_KERNEL); + if (!hash_data->hashtab) { + *ret = -ENOMEM; + goto free; + } + + for (i = 0; i < hashtab_size; i++) + INIT_HLIST_HEAD(&hash_data->hashtab[i]); + spin_lock_init(&hash_data->lock); + + hash_data->hashtab_bits = hashtab_bits; + hash_data->event_file = file; + out: + return hash_data; + free: + destroy_hashdata(hash_data); + hash_data = NULL; + goto out; +} + +static inline bool match_stacktraces(struct stack_trace *entry_stacktrace, + struct stack_trace *stacktrace) +{ + unsigned int size; + + if (entry_stacktrace->nr_entries != entry_stacktrace->nr_entries) + return false; + + size = sizeof(*stacktrace->entries) * stacktrace->nr_entries; + if (memcmp(entry_stacktrace->entries, stacktrace->entries, size) == 0) + return true; + + return false; +} + +static struct hash_trigger_entry * +hash_trigger_entry_match(struct hash_trigger_entry *entry, + struct hash_key_part *key_parts, + unsigned int n_key_parts) +{ + unsigned int i; + + for (i = 0; i < n_key_parts; i++) { + if (entry->key_parts[i].type != key_parts[i].type) + return NULL; + + switch (entry->key_parts[i].type) { + case HASH_KEY_TYPE_U64: + if (entry->key_parts[i].var.val_u64 != + key_parts[i].var.val_u64) + return NULL; + break; + case HASH_KEY_TYPE_STACKTRACE: + if (!match_stacktraces(entry->key_parts[i].var.val_stacktrace, + key_parts[i].var.val_stacktrace)) + return NULL; + break; + case HASH_KEY_TYPE_STRING: + if (strcmp(entry->key_parts[i].var.val_string, + key_parts[i].var.val_string)) + return NULL; + break; + default: + return NULL; + } + } + + return entry; +} + +static struct hash_trigger_entry * +hash_trigger_entry_find(struct hash_trigger_data *hash_data, void *rec, + struct stack_trace *stacktrace) +{ + struct hash_key_part key_parts[COMPOUND_KEY_MAX]; + unsigned int i, n_keys = hash_data->n_keys; + struct hash_trigger_entry *entry; + struct hash_field *hash_field; + u64 hash_idx; + + hash_idx = hash_key(hash_data, rec, stacktrace); + + for (i = 0; i < n_keys; i++) { + hash_field = hash_data->keys[i]; + if (hash_field->flags & HASH_FIELD_STACKTRACE) { + key_parts[i].type = HASH_KEY_TYPE_STACKTRACE; + key_parts[i].var.val_stacktrace = stacktrace; + } else if (hash_field->flags & HASH_FIELD_STRING) { + u64 hash_val = hash_field->fn(hash_field, rec); + + key_parts[i].type = HASH_KEY_TYPE_STRING; + key_parts[i].var.val_string = (char *)hash_val; + } else { + u64 hash_val = hash_field->fn(hash_field, rec); + + key_parts[i].type = HASH_KEY_TYPE_U64; + key_parts[i].var.val_u64 = hash_val; + } + } + + hlist_for_each_entry_rcu(entry, &hash_data->hashtab[hash_idx], node) { + if (hash_trigger_entry_match(entry, key_parts, n_keys)) + return entry; + } + + return NULL; +} + +static void hash_trigger_entry_insert(struct hash_trigger_data *hash_data, + struct hash_trigger_entry *entry, + void *rec, + struct stack_trace *stacktrace) +{ + u64 hash_idx = hash_key(hash_data, rec, stacktrace); + + hash_data->total_entries++; + + hlist_add_head_rcu(&entry->node, &hash_data->hashtab[hash_idx]); +} + +static void +hash_trigger_entry_update(struct hash_trigger_data *hash_data, + struct hash_trigger_entry *entry, void *rec) +{ + struct hash_field *hash_field; + unsigned int i; + u64 hash_val; + + for (i = 0; i < hash_data->n_vals; i++) { + hash_field = hash_data->vals[i]; + hash_val = hash_field->fn(hash_field, rec); + entry->sums[i] += hash_val; + } + + entry->count++; +} + +static void +event_hash_trigger(struct event_trigger_data *data, void *rec) +{ + struct hash_trigger_data *hash_data = data->private_data; + struct hash_trigger_entry *entry; + struct hash_field *hash_field; + + struct stack_trace stacktrace; + unsigned long entries[HASH_STACKTRACE_DEPTH]; + + unsigned long flags; + + if (hash_data->drops) { + hash_data->drops++; + return; + } + + hash_field = hash_data->keys[0]; + + if (hash_field->flags & HASH_FIELD_STACKTRACE) { + stacktrace.max_entries = HASH_STACKTRACE_DEPTH; + stacktrace.entries = entries; + stacktrace.nr_entries = 0; + stacktrace.skip = HASH_STACKTRACE_SKIP; + + save_stack_trace(&stacktrace); + } + + spin_lock_irqsave(&hash_data->lock, flags); + entry = hash_trigger_entry_find(hash_data, rec, &stacktrace); + + if (!entry) { + entry = hash_trigger_entry_create(hash_data, rec, &stacktrace); + WARN_ON_ONCE(!entry); + if (!entry) { + spin_unlock_irqrestore(&hash_data->lock, flags); + return; + } + hash_trigger_entry_insert(hash_data, entry, rec, &stacktrace); + } + + hash_trigger_entry_update(hash_data, entry, rec); + hash_data->total_hits++; + spin_unlock_irqrestore(&hash_data->lock, flags); +} + +static void +hash_trigger_stacktrace_print(struct seq_file *m, + struct stack_trace *stacktrace) +{ + char str[KSYM_SYMBOL_LEN]; + unsigned int spaces = 8; + unsigned int i; + + for (i = 0; i < stacktrace->nr_entries; i++) { + if (stacktrace->entries[i] == ULONG_MAX) + return; + seq_printf(m, "%*c", 1 + spaces, ' '); + sprint_symbol(str, stacktrace->entries[i]); + seq_printf(m, "%s\n", str); + } +} + +static void +hash_trigger_entry_print(struct seq_file *m, + struct hash_trigger_data *hash_data, + struct hash_trigger_entry *entry) +{ + char str[KSYM_SYMBOL_LEN]; + unsigned int i; + + seq_printf(m, "key: "); + for (i = 0; i < hash_data->n_keys; i++) { + if (i > 0) + seq_printf(m, ", "); + if (entry->key_parts[i].flags & HASH_FIELD_SYM) { + kallsyms_lookup(entry->key_parts[i].var.val_u64, + NULL, NULL, NULL, str); + seq_printf(m, "%s:[%llx] %s", + hash_data->keys[i]->field->name, + entry->key_parts[i].var.val_u64, + str); + } else if (entry->key_parts[i].flags & HASH_FIELD_HEX) { + seq_printf(m, "%s:%llx", + hash_data->keys[i]->field->name, + entry->key_parts[i].var.val_u64); + } else if (entry->key_parts[i].flags & HASH_FIELD_STACKTRACE) { + seq_printf(m, "stacktrace:\n"); + hash_trigger_stacktrace_print(m, + entry->key_parts[i].var.val_stacktrace); + } else if (entry->key_parts[i].flags & HASH_FIELD_STRING) { + seq_printf(m, "%s:%s", + hash_data->keys[i]->field->name, + entry->key_parts[i].var.val_string); + } else if (entry->key_parts[i].flags & HASH_FIELD_EXECNAME) { + seq_printf(m, "%s:%s[%llu]", + hash_data->keys[i]->field->name, + entry->comm, + entry->key_parts[i].var.val_u64); + } else if (entry->key_parts[i].flags & HASH_FIELD_SYSCALL) { + int syscall = entry->key_parts[i].var.val_u64; + const char *syscall_name = get_syscall_name(syscall); + + if (!syscall_name) + syscall_name = "unknown_syscall"; + seq_printf(m, "%s:%s", + hash_data->keys[i]->field->name, + syscall_name); + } else { + seq_printf(m, "%s:%llu", + hash_data->keys[i]->field->name, + entry->key_parts[i].var.val_u64); + } + } + + seq_printf(m, "\tvals: count:%llu", entry->count); + + for (i = 0; i < hash_data->n_vals; i++) { + if (i > 0) + seq_printf(m, ", "); + if (hash_data->vals[i]->aux_field) { + seq_printf(m, " %s-%s:%llu", + hash_data->vals[i]->field->name, + hash_data->vals[i]->aux_field->name, + entry->sums[i]); + continue; + } + seq_printf(m, " %s:%llu", + hash_data->vals[i]->field->name, + entry->sums[i]); + } + seq_printf(m, "\n"); +} + +static int sort_entries(const struct hash_trigger_entry **a, + const struct hash_trigger_entry **b) +{ + const struct hash_trigger_entry *entry_a, *entry_b; + struct hash_trigger_sort_key *sort_key; + struct hash_trigger_data *hash_data; + u64 val_a, val_b; + int ret = 0; + + entry_a = *a; + entry_b = *b; + + hash_data = entry_a->hash_data; + sort_key = hash_data->sort_key_cur; + + if (sort_key->use_hitcount) { + val_a = entry_a->count; + val_b = entry_b->count; + } else if (sort_key->key_part) { + /* TODO: make sure we never use a stacktrace here */ + val_a = entry_a->key_parts[sort_key->idx].var.val_u64; + val_b = entry_b->key_parts[sort_key->idx].var.val_u64; + } else { + val_a = entry_a->sums[sort_key->idx]; + val_b = entry_b->sums[sort_key->idx]; + } + + if (val_a > val_b) + ret = 1; + else if (val_a < val_b) + ret = -1; + + if (sort_key->descending) + ret = -ret; + + return ret; +} + +static void sort_secondary(struct hash_trigger_data *hash_data, + struct hash_trigger_entry **entries, + unsigned int n_entries) +{ + struct hash_trigger_sort_key *primary_sort_key; + unsigned int start = 0, n_subelts = 1; + struct hash_trigger_entry *entry; + bool do_sort = false; + unsigned int i, idx; + u64 cur_val; + + primary_sort_key = hash_data->sort_keys[0]; + + entry = entries[0]; + if (primary_sort_key->use_hitcount) + cur_val = entry->count; + else if (primary_sort_key->key_part) + cur_val = entry->key_parts[primary_sort_key->idx].var.val_u64; + else + cur_val = entry->sums[primary_sort_key->idx]; + + hash_data->sort_key_cur = hash_data->sort_keys[1]; + + for (i = 1; i < n_entries; i++) { + entry = entries[i]; + if (primary_sort_key->use_hitcount) { + if (entry->count != cur_val) { + cur_val = entry->count; + do_sort = true; + } + } else if (primary_sort_key->key_part) { + idx = primary_sort_key->idx; + if (entry->key_parts[idx].var.val_u64 != cur_val) { + cur_val = entry->key_parts[idx].var.val_u64; + do_sort = true; + } + } else { + idx = primary_sort_key->idx; + if (entry->sums[idx] != cur_val) { + cur_val = entry->sums[idx]; + do_sort = true; + } + } + + if (i == n_entries - 1) + do_sort = true; + + if (do_sort) { + if (n_subelts > 1) { + sort(entries + start, n_subelts, sizeof(entry), + (int (*)(const void *, const void *))sort_entries, NULL); + } + start = i; + n_subelts = 1; + do_sort = false; + } else + n_subelts++; + } +} + +static bool +print_entries_sorted(struct seq_file *m, struct hash_trigger_data *hash_data) +{ + unsigned int hashtab_size = (1 << hash_data->hashtab_bits); + struct hash_trigger_entry **entries; + struct hash_trigger_entry *entry; + unsigned int entries_size; + unsigned int i = 0, j = 0; + + entries_size = sizeof(entry) * hash_data->total_entries; + entries = kmalloc(entries_size, GFP_KERNEL); + if (!entries) + return false; + + for (i = 0; i < hashtab_size; i++) { + hlist_for_each_entry_rcu(entry, &hash_data->hashtab[i], node) + entries[j++] = entry; + } + + hash_data->sort_key_cur = hash_data->sort_keys[0]; + sort(entries, j, sizeof(struct hash_trigger_entry *), + (int (*)(const void *, const void *))sort_entries, NULL); + + if (hash_data->sort_keys[1]) + sort_secondary(hash_data, entries, j); + + for (i = 0; i < j; i++) + hash_trigger_entry_print(m, hash_data, entries[i]); + + kfree(entries); + + return true; +} + +static bool +print_entries_unsorted(struct seq_file *m, struct hash_trigger_data *hash_data) +{ + unsigned int hashtab_size = (1 << hash_data->hashtab_bits); + struct hash_trigger_entry *entry; + unsigned int i = 0; + + for (i = 0; i < hashtab_size; i++) { + hlist_for_each_entry_rcu(entry, &hash_data->hashtab[i], node) + hash_trigger_entry_print(m, hash_data, entry); + } + + return true; +} + +static int +event_hash_trigger_print(struct seq_file *m, struct event_trigger_ops *ops, + struct event_trigger_data *data) +{ + struct hash_trigger_data *hash_data = data->private_data; + bool sorted; + int ret; + + ret = event_trigger_print("hash", m, (void *)data->count, + data->filter_str); + + sorted = print_entries_sorted(m, hash_data); + if (!sorted) + print_entries_unsorted(m, hash_data); + + seq_printf(m, "Totals:\n Hits: %lu\n Entries: %lu\n Dropped: %lu\n", + hash_data->total_hits, hash_data->total_entries, hash_data->drops); + + if (!sorted) + seq_printf(m, "Unsorted (couldn't alloc memory for sorting)\n"); + + return ret; +} + +static void +event_hash_trigger_free(struct event_trigger_ops *ops, + struct event_trigger_data *data) +{ + struct hash_trigger_data *hash_data = data->private_data; + + if (WARN_ON_ONCE(data->ref <= 0)) + return; + + data->ref--; + if (!data->ref) { + destroy_hashdata(hash_data); + trigger_data_free(data); + } +} + +static struct event_trigger_ops event_hash_trigger_ops = { + .func = event_hash_trigger, + .print = event_hash_trigger_print, + .init = event_trigger_init, + .free = event_hash_trigger_free, +}; + +static struct event_trigger_ops * +event_hash_get_trigger_ops(char *cmd, char *param) +{ + /* counts don't make sense for hash triggers */ + return &event_hash_trigger_ops; +} + +static int +event_hash_trigger_func(struct event_command *cmd_ops, + struct ftrace_event_file *file, + char *glob, char *cmd, char *param) +{ + struct event_trigger_data *trigger_data; + struct event_trigger_ops *trigger_ops; + struct hash_trigger_data *hash_data; + char *sort_keys = NULL; + char *trigger; + char *number; + int ret = 0; + char *keys; + char *vals; + + if (!param) + return -EINVAL; + + /* separate the trigger from the filter (s:e:n [if filter]) */ + trigger = strsep(¶m, " \t"); + if (!trigger) + return -EINVAL; + + keys = strsep(&trigger, ":"); + if (!trigger) + return -EINVAL; + + vals = strsep(&trigger, ":"); + if (trigger) + sort_keys = strsep(&trigger, ":"); + + hash_data = create_hash_data(HASH_TRIGGER_BITS, keys, vals, sort_keys, + file, &ret); + if (ret) + return ret; + + trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger); + + ret = -ENOMEM; + trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); + if (!trigger_data) + goto out; + + trigger_data->count = -1; + trigger_data->ops = trigger_ops; + trigger_data->cmd_ops = cmd_ops; + INIT_LIST_HEAD(&trigger_data->list); + RCU_INIT_POINTER(trigger_data->filter, NULL); + + trigger_data->private_data = hash_data; + + if (glob[0] == '!') { + cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); + ret = 0; + goto out_free; + } + + if (trigger) { + number = strsep(&trigger, ":"); + + ret = -EINVAL; + if (strlen(number)) /* hash triggers don't support counts */ + goto out_free; + } + + if (!param) /* if param is non-empty, it's supposed to be a filter */ + goto out_reg; + + if (!cmd_ops->set_filter) + goto out_reg; + + ret = cmd_ops->set_filter(param, trigger_data, file); + if (ret < 0) + goto out_free; + + out_reg: + ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file); + /* + * The above returns on success the # of functions enabled, + * but if it didn't find any functions it returns zero. + * Consider no functions a failure too. + */ + if (!ret) { + ret = -ENOENT; + goto out_free; + } else if (ret < 0) + goto out_free; + /* Just return zero, not the number of enabled functions */ + ret = 0; + out: + return ret; + + out_free: + if (cmd_ops->set_filter) + cmd_ops->set_filter(NULL, trigger_data, NULL); + kfree(trigger_data); + destroy_hashdata(hash_data); + goto out; +} + +static struct event_command trigger_hash_cmd= { + .name = "hash", + .trigger_type = ETT_EVENT_HASH, + .post_trigger = true, /* need non-NULL rec */ + .func = event_hash_trigger_func, + .reg = register_trigger, + .unreg = unregister_trigger, + .get_trigger_ops = event_hash_get_trigger_ops, + .set_filter = set_trigger_filter, +}; + +static __init int register_trigger_hash_cmd(void) +{ + int ret; + + ret = register_event_command(&trigger_hash_cmd); + WARN_ON(ret < 0); + + return ret; +} + __init int register_trigger_cmds(void) { register_trigger_traceon_traceoff_cmds(); register_trigger_snapshot_cmd(); register_trigger_stacktrace_cmd(); register_trigger_enable_disable_cmds(); + register_trigger_hash_cmd(); return 0; } |