aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Documentation/trace/hist-triggers-test.txt743
1 files changed, 743 insertions, 0 deletions
diff --git a/Documentation/trace/hist-triggers-test.txt b/Documentation/trace/hist-triggers-test.txt
new file mode 100644
index 000000000000..2afe2955ceaa
--- /dev/null
+++ b/Documentation/trace/hist-triggers-test.txt
@@ -0,0 +1,743 @@
+####
+
+Documentation
+
+echo 'hist:key=call_site:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+echo '!hist:key=call_site:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+echo 'hist:key=call_site.hex:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+echo '!hist:key=call_site.hex:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+echo 'hist:key=call_site.sym:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+echo '!hist:key=call_site.sym:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+echo 'hist:key=call_site.sym:val=bytes_req:sort=bytes_req.descending' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+echo '!hist:key=call_site.sym:val=bytes_req:sort=bytes_req.descending' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+echo 'hist:keys=call_site.sym:values=bytes_req,bytes_alloc:sort=bytes_alloc.descending' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+echo '!hist:keys=call_site.sym:values=bytes_req,bytes_alloc:sort=bytes_alloc.descending' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+echo 'hist:keys=call_site.sym-offset:values=bytes_req,bytes_alloc:sort=bytes_alloc.descending' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+echo '!hist:keys=call_site.sym-offset:values=bytes_req,bytes_alloc:sort=bytes_alloc.descending' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+echo 'hist:keys=stacktrace:values=bytes_req,bytes_alloc:sort=bytes_alloc' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+echo '!hist:keys=stacktrace:values=bytes_req,bytes_alloc:sort=bytes_alloc' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+echo 'hist:keys=stacktrace:values=bytes_req,bytes_alloc:sort=bytes_alloc.descending' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+echo '!hist:keys=stacktrace:values=bytes_req,bytes_alloc:sort=bytes_alloc.descending' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+echo 'hist:keys=stacktrace:values=hitcount' > /sys/kernel/debug/tracing/events/kmem/kfree/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kfree/hist
+echo '!hist:keys=stacktrace:values=hitcount' > /sys/kernel/debug/tracing/events/kmem/kfree/trigger
+
+echo 'hist:key=common_pid.execname:val=count:sort=count' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/trigger
+cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/hist
+echo '!hist:key=common_pid.execname:val=count:sort=count' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/trigger
+
+echo 'hist:key=id.syscall:val=hitcount' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+cat /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/hist
+echo '!hist:key=id.syscall:val=hitcount' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+
+# for slob, need to use kmalloc_node, no trace_kmalloc, but trace_kmalloc_node
+echo 'hist:keys=stacktrace:values=bytes_req,bytes_alloc:sort=bytes_alloc' > /sys/kernel/debug/tracing/events/kmem/kmalloc_node/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc_node/hist
+echo '!hist:keys=stacktrace:values=bytes_req,bytes_alloc:sort=bytes_alloc' > /sys/kernel/debug/tracing/events/kmem/kmalloc_node/trigger
+
+echo 'hist:key=child_comm:val=hitcount:size=256' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_fork/hist
+echo 'hist:key=child_comm:val=hitcount:pause' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_fork/hist
+echo 'hist:key=child_comm:val=hitcount:cont' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+echo '!hist:key=child_comm:val=hitcount:size=256' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+
+echo 'hist:keys=skbaddr.hex:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:key=stacktrace:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'enable_hist:net:netif_receive_skb if filename==/usr/bin/wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+echo 'disable_hist:net:netif_receive_skb if comm==wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exit/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exit/trigger
+
+wget https://www.kernel.org/pub/linux/kernel/v3.x/patch-3.19.xz
+wget https://www.kernel.org/pub/linux/kernel/v3.x/linux-3.19.tar.xz
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+echo 'enable_event:net:netif_receive_skb if filename==/usr/bin/wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+echo 'disable_event:net:netif_receive_skb if comm==wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exit/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exit/trigger
+
+echo 'hist:keys=skbaddr.hex:vals=len:clear' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+echo 'hist:key=stacktrace:vals=len:clear' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+echo > /sys/kernel/debug/tracing/trace
+
+wget https://www.kernel.org/pub/linux/kernel/v3.x/patch-3.19.xz
+wget https://www.kernel.org/pub/linux/kernel/v3.x/linux-3.19.tar.xz
+
+cat /sys/kernel/debug/tracing/trace
+
+echo '!hist:keys=skbaddr.hex:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:key=stacktrace:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!enable_hist:net:netif_receive_skb if filename==/usr/bin/wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+echo '!disable_hist:net:netif_receive_skb if comm==wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exit/trigger
+echo '!enable_event:net:netif_receive_skb if filename==/usr/bin/wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+echo '!disable_event:net:netif_receive_skb if comm==wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exit/trigger
+
+# multiple triggers for same event, different filter and fields
+echo 'hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=len:vals=common_preempt_count' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+echo '!hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=len:vals=common_preempt_count' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+#named triggers
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist; cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# named triggers without any fields, you can combine anything
+echo 'hist:name=bar:key=stacktrace:val=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+echo 'hist:name=bar:key=stacktrace:val=hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_fork/hist
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:name=bar:key=stacktrace:val=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+echo '!hist:name=bar:key=stacktrace:val=hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+####
+
+#named triggers
+
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist;cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+##
+
+# filters can be different for same named filter
+echo 'hist:name=foo:key=call_site:val=bytes_req if bytes_req > 0' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+echo 'hist:name=foo:key=call_site:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc_node/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc_node/hist
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc_node/trigger
+
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist;cat /sys/kernel/debug/tracing/events/kmem/kmalloc_node/hist
+
+echo '!hist:name=foo:key=call_site:val=bytes_req if bytes_req > 0' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+echo '!hist:name=foo:key=call_site:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc_node/trigger
+
+##
+
+# without any fields you can combine anything
+echo 'hist:name=foo:key=stacktrace:val=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+echo 'hist:name=foo:key=stacktrace:val=hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_fork/hist
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:name=foo:key=stacktrace:val=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+echo '!hist:name=foo:key=stacktrace:val=hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+##
+
+# but with fields you should get error
+echo 'hist:name=foo:key=skbaddr:val=hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:key=stacktrace:val=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+echo '!hist:name=foo:key=skbaddr:val=hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo 'hist:name=foo:key=stacktrace:val=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:key=stacktrace:val=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+echo '!hist:name=foo:key=stacktrace:val=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+##
+
+# named and unnamed different namespace, same trigger but one named the other not
+
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:cont' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:cont' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo 'hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo 'hist:keys=skbaddr.hex:vals=len:pause if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:pause if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:cont if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:cont if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# make sure you still get existing error for same event whether or not named
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+##
+
+# named and unnamed different namespace, same triggers but 2 named the other not
+
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:cont' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len:cont' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:cont' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo 'hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo 'hist:keys=skbaddr.hex:vals=len:pause if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:pause if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len:pause if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:cont if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:cont if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len:cont if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# make sure you still get existing error for same event whether or not named
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:name=bar:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+##
+
+# make sure you still get existing error for same event whether or not named
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# EEXIST
+echo 'hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo '!hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:name=bar:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+##
+
+echo 'hist:keys=skbaddr.hex:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo 'hist:keys=skbaddr.hex:vals=len:cont' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:pause if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:cont if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+####
+
+# multiple triggers for same event, different filter and fields
+
+echo 'hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=len:vals=common_preempt_count' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+echo '!hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=len:vals=common_preempt_count' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+####
+
+# multiple named triggers for same event, filter first then non-filter
+
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+# multiple named triggers for same event, non-filter first then filter
+
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+# multiple non-named triggers for same event, filter first then non-filter
+
+echo 'hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+echo '!hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+# multiple non-named triggers for same event, non-filter first then filter
+
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+echo '!hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+##
+
+# multiple named triggers for same event, different filter
+
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+##
+
+# multiple triggers for same event, different filter, pause, cont, clear
+
+echo 'hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+# pause
+echo 'hist:keys=skbaddr.hex:vals=len:pause if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+#clear, pauses all but should leave previous paused
+echo 'hist:keys=skbaddr.hex:vals=len:clear if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+# cont
+echo 'hist:keys=skbaddr.hex:vals=len:cont if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+echo '!hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+##
+
+# multiple named triggers for same event, different filter, pause, cont, clear
+
+# named triggers should be the same as non-named for pause and cont i.e. each must be specified explicitly for pause and cont
+# for clear, clearing one named trigger clears all of the same name, since they reference the same hist
+
+# multiple named triggers for same event, different filter, pause, cont, clear
+
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+# pause
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:pause if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+#clear, pauses all but should leave previous paused
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:clear if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+# cont
+echo 'hist:name=foo:keys=skbaddr.hex:vals=len:cont if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:name=foo:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+##
+
+echo 'hist:key=common_pid.execname if ret > 0' > /sys/kernel/debug/tracing/events/syscalls/sys_exit_write/trigger
+echo 'hist:key=common_pid.execname if ret == 0' > /sys/kernel/debug/tracing/events/syscalls/sys_exit_write/trigger
+echo 'hist:key=common_pid.execname if ret < 0' > /sys/kernel/debug/tracing/events/syscalls/sys_exit_write/trigger
+
+cat /sys/kernel/debug/tracing/events/syscalls/sys_exit_write/trigger
+cat /sys/kernel/debug/tracing/events/syscalls/sys_exit_write/hist
+
+echo '!hist:key=common_pid.execname if ret > 0' > /sys/kernel/debug/tracing/events/syscalls/sys_exit_write/trigger
+echo '!hist:key=common_pid.execname if ret == 0' > /sys/kernel/debug/tracing/events/syscalls/sys_exit_write/trigger
+echo '!hist:key=common_pid.execname if ret < 0' > /sys/kernel/debug/tracing/events/syscalls/sys_exit_write/trigger
+
+####
+
+# enable_hist on multiple triggers for same event, different filter
+
+echo 'hist:keys=skbaddr.hex:vals=len:pause if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:pause if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:pause if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:pause' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+echo 'enable_hist:net:netif_receive_skb if filename==/usr/bin/wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+echo 'disable_hist:net:netif_receive_skb if comm==wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exit/trigger
+
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exit/trigger
+
+wget https://www.kernel.org/pub/linux/kernel/v3.x/patch-3.19.xz
+wget https://www.kernel.org/pub/linux/kernel/v3.x/linux-3.19.tar.xz
+
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+
+echo '!enable_hist:net:netif_receive_skb if filename==/usr/bin/wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+echo '!disable_hist:net:netif_receive_skb if comm==wget' > /sys/kernel/debug/tracing/events/sched/sched_process_exit/trigger
+
+echo '!hist:keys=skbaddr.hex:vals=len if len < 0' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len if len > 4096' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len if len == 256' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+####
+
+mkdir /sys/kernel/debug/tracing/instances/foo
+
+echo 'hist:key=call_site.hex:val=bytes_req' > /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/trigger
+echo 'hist:key=call_site.hex:val=bytes_req if bytes_req > 256' > /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/trigger
+echo 'hist:name=foo:key=call_site.hex:val=bytes_req' > /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/trigger
+echo 'hist:name=foo:key=call_site.hex:val=bytes_req if bytes_req > 256' > /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/hist
+cat /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/trigger
+
+echo 'hist:key=call_site.hex:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+echo 'hist:key=call_site.hex:val=bytes_req if bytes_req > 512' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+echo 'hist:name=foo:key=call_site.hex:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+echo 'hist:name=foo:key=call_site.hex:val=bytes_req if bytes_req > 512' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+cat /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/hist;cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+
+echo '!hist:key=call_site.hex:val=bytes_req' > /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/trigger
+echo '!hist:key=call_site.hex:val=bytes_req if bytes_req > 256' > /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/trigger
+echo '!hist:name=foo:key=call_site.hex:val=bytes_req' > /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/trigger
+echo '!hist:name=foo:key=call_site.hex:val=bytes_req if bytes_req > 256' > /sys/kernel/debug/tracing/instances/foo/events/kmem/kmalloc/trigger
+echo '!hist:key=call_site.hex:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+echo '!hist:key=call_site.hex:val=bytes_req if bytes_req > 512' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+echo '!hist:name=foo:key=call_site.hex:val=bytes_req' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+echo '!hist:name=foo:key=call_site.hex:val=bytes_req if bytes_req > 512' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+####
+
+# default e.g. no sort key = hitcount
+echo 'hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# single sort key
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:keys=skbaddr.hex:vals=len:sort=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# single sort key descending
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len.descending' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:keys=skbaddr.hex:vals=len:sort=len.descending' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# same sort key
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len,len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:keys=skbaddr.hex:vals=len:sort=len,len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# too many and same sort key
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len,len,len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# too many and different sort key
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len,hitcount,len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# compound key, multiple sort keys, high volume
+echo 'hist:key=common_pid.execname,id.syscall:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+cat /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/hist
+echo '!hist:key=common_pid.execname,id.syscall:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+
+# compound key, multiple sort keys, high volume, find highest of each syscall
+echo 'hist:key=id.syscall,common_pid.execname:val=hitcount:sort=id,hitcount' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+cat /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/hist
+echo '!hist:key=id.syscall,common_pid.execname:val=hitcount:sort=id,hitcount' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+
+# compound key, multiple sort keys, high volume, find highest of each syscall, descending sort
+echo 'hist:key=id,common_pid:val=hitcount:sort=id.descending,hitcount.descending' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+cat /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/hist
+echo '!hist:key=id,common_pid:val=hitcount:sort=id.descending,hitcount.descending' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+
+# compound key, multiple sort keys, high volume, find highest of one kind of syscall
+echo 'hist:key=id.syscall,common_pid.execname:val=hitcount:sort=id,hitcount if id == 16' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+cat /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/hist
+echo '!hist:key=id.syscall,common_pid.execname:val=hitcount:sort=id,hitcount if id == 16' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+
+# compound key, multiple sort keys, low volume
+echo 'hist:key=common_pid.execname,size:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/trigger
+cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/hist
+echo '!hist:key=common_pid.execname,size:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/trigger
+
+# compound key, multiple sort keys, low volume, reverse keys
+echo 'hist:key=size,common_pid.execname:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/trigger
+cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/hist
+echo '!hist:key=size,common_pid.execname:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/trigger
+
+# compound key, multiple sort keys - both sort keys are keys
+echo 'hist:key=common_pid.execname,size:val=hitcount:sort=common_pid,size' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/trigger
+cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/hist
+echo '!hist:key=common_pid.execname,size:val=hitcount:sort=common_pid,size' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/trigger
+
+# compound key, multiple sort keys - both sort keys are NOT keys
+echo 'hist:key=common_pid.execname,len:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:key=common_pid.execname,len:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# compound key, multiple sort keys - both sort keys are keys
+echo 'hist:key=common_pid.execname,len:val=hitcount:sort=common_pid,len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:key=common_pid.execname,len:val=hitcount:sort=common_pid,len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# compound key, multiple sort keys - both sort keys are keys, reverse keys
+echo 'hist:key=size,common_pid.execname:val=hitcount:sort=common_pid,size' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/trigger
+cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/hist
+echo '!hist:key=size,common_pid.execname:val=hitcount:sort=common_pid,size' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_recvfrom/trigger
+
+# compound key, multiple sort keys
+echo 'hist:key=common_pid.execname,len:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_mmap/trigger
+cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_mmap/hist
+echo '!hist:key=common_pid.execname,len:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_mmap/trigger
+
+# compound key, multiple sort keys, -EINVAL because only common_pid can have execname
+echo 'hist:key=common_pid.execname,parent_pid.execname:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_fork/trigger
+
+# compound key, multiple sort keys, reversed key order, for testing offset
+echo 'hist:key=id.syscall,common_pid.execname:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+cat /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/hist
+echo '!hist:key=id.syscall,common_pid.execname:val=hitcount:sort=common_pid,hitcount' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+
+# compound key, stacktrace in key
+echo 'hist:key=stacktrace,len:val=hitcount:sort=len,hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:key=stacktrace,len:val=hitcount:sort=len,hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# compound key, stacktrace second in key
+echo 'hist:key=len,stacktrace:val=hitcount:sort=len,hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+echo '!hist:key=len,stacktrace:val=hitcount:sort=len,hitcount' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+####
+
+# should produce EINVAL errors
+
+echo 'hist:keys=skbaddr.hex:vals=len:sort' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:sort=' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:sort=,' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len,' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len.' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len.ascendin' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len.descendin' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:key=id,common_pid:val=hitcount:sort=id.descendin,hitcount.descending' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+echo 'hist:key=id,common_pid:val=hitcount:sort=id.descending,hitcount.descendin' > /sys/kernel/debug/tracing/events/raw_syscalls/sys_enter/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len,len,len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# too many keys
+echo 'hist:keys=filename,common_pid,stacktrace:vals=hitcount:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+# too many vals
+echo 'hist:keys=filename:vals=common_pid,pid,old_pid:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+# too many vals, hitcount first
+echo 'hist:keys=filename:vals=hitcount,common_pid,pid,old_pid:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+# too many vals, hitcount first, take out hitcount and works because hitcount automatic, shows max applies to specified
+echo 'hist:keys=filename:vals=hitcount,pid,old_pid:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+# key size too big (> string + stacktrace)
+echo 'hist:keys=filename,filename:vals=hitcount:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+# should NOT produce EINVAL errors
+
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len.ascending' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len.descending' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=skbaddr.hex:vals=len:sort=len,len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo 'hist:keys=filename:vals=hitcount,common_pid:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+echo 'hist:keys=filename:vals=pid,old_pid:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+cat /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exec/hist
+
+# hex on value
+echo 'hist:keys=skbaddr.hex:vals=len.hex:sort=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+
+# sym on value (EINVAL)
+echo 'hist:keys=skbaddr.hex:vals=len.sym:sort=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+####
+
+#test illegal sort on stacktrace
+echo 'hist:keys=stacktrace:vals=len:sort=stacktrace' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# test sort on key
+echo 'hist:key=common_pid:val=count:sort=common_pid' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/trigger
+cat /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/hist
+echo '!hist:key=common_pid:val=count:sort=common_pid' > /sys/kernel/debug/tracing/events/syscalls/sys_enter_read/trigger
+
+# variable length string
+echo 'hist:keys=filename:vals=hitcount:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exec/hist
+echo '!hist:keys=filename:vals=hitcount:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+# variable length string as first field in key
+echo 'hist:keys=filename,common_pid:vals=hitcount:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exec/hist
+echo '!hist:keys=filename,common_pid:vals=hitcount:sort=hitcount' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+# variable length string as first field in key, sort on string and secondary descending
+echo 'hist:keys=filename,common_pid:vals=hitcount:sort=filename,common_pid.descending' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exec/hist
+echo '!hist:keys=filename,common_pid:vals=hitcount:sort=filename,common_pid.descending' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+# variable length string as first field in key, sort on string
+echo 'hist:keys=filename,common_pid:vals=hitcount:sort=filename' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exec/hist
+echo '!hist:keys=filename,common_pid:vals=hitcount:sort=filename' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+# variable length string as last field in key, first field char, sort on string and secondary
+echo 'hist:keys=common_preempt_count,filename:vals=hitcount:sort=common_preempt_count,filename' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exec/hist
+echo '!hist:keys=common_preempt_count,filename:vals=hitcount:sort=common_preempt_count,filename' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+# sort on string
+echo 'hist:keys=filename:vals=hitcount:sort=filename' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+cat /sys/kernel/debug/tracing/events/sched/sched_process_exec/hist
+echo '!hist:keys=filename:vals=hitcount:sort=filename' > /sys/kernel/debug/tracing/events/sched/sched_process_exec/trigger
+
+####
+
+# More strings
+
+echo 'hist:keys=name:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_receive_skb/hist
+echo '!hist:keys=name:vals=len' > /sys/kernel/debug/tracing/events/net/netif_receive_skb/trigger
+
+echo 'hist:keys=oldcomm:vals=hitcount:sort=oldcomm' > /sys/kernel/debug/tracing/events/task/task_rename/trigger
+cat /sys/kernel/debug/tracing/events/task/task_rename/hist
+echo '!hist:keys=oldcomm:vals=hitcount:sort=oldcomm' > /sys/kernel/debug/tracing/events/task/task_rename/trigger
+
+####
+
+# max tests
+
+# EINVAL because size too small
+echo 'hist:keys=skbaddr.hex:vals=len:size=8' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+# smallest size
+echo 'hist:keys=skbaddr.hex:vals=len:size=128' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+cat /sys/kernel/debug/tracing/events/net/netif_rx/hist
+cat /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+echo '!hist:keys=skbaddr.hex:vals=len' > /sys/kernel/debug/tracing/events/net/netif_rx/trigger
+
+echo 'hist:key=pfn:val=hitcount' > /sys/kernel/debug/tracing/events/kmem/mm_page_alloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/mm_page_alloc/hist
+echo '!hist:key=pfn:val=hitcount' > /sys/kernel/debug/tracing/events/kmem/mm_page_alloc/trigger
+
+echo 'hist:key=ptr.hex:val=hitcount' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+echo '!hist:key=ptr.hex:val=hitcount' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+echo 'hist:keys=stacktrace:vals=bytes_req,bytes_alloc:sort=bytes_alloc:size=2048' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+echo 'hist:key=ptr:val=hitcount' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+cat /sys/kernel/debug/tracing/events/kmem/kmalloc/hist
+echo '!hist:keys=stacktrace:vals=bytes_req,bytes_alloc:sort=bytes_alloc:size=2048' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+echo '!hist:key=ptr:val=hitcount' > /sys/kernel/debug/tracing/events/kmem/kmalloc/trigger
+
+####
+
+# function hist
+
+echo function_hist > /sys/kernel/debug/tracing/current_tracer
+cat /sys/kernel/debug/tracing/function_hist
+