aboutsummaryrefslogtreecommitdiffstats
path: root/common/recipes-kernel/linux/linux-yocto-4.9.21/0007-bpf-x64-implement-retpoline-for-tail-call.patch
diff options
context:
space:
mode:
Diffstat (limited to 'common/recipes-kernel/linux/linux-yocto-4.9.21/0007-bpf-x64-implement-retpoline-for-tail-call.patch')
-rw-r--r--common/recipes-kernel/linux/linux-yocto-4.9.21/0007-bpf-x64-implement-retpoline-for-tail-call.patch183
1 files changed, 183 insertions, 0 deletions
diff --git a/common/recipes-kernel/linux/linux-yocto-4.9.21/0007-bpf-x64-implement-retpoline-for-tail-call.patch b/common/recipes-kernel/linux/linux-yocto-4.9.21/0007-bpf-x64-implement-retpoline-for-tail-call.patch
new file mode 100644
index 00000000..69809c28
--- /dev/null
+++ b/common/recipes-kernel/linux/linux-yocto-4.9.21/0007-bpf-x64-implement-retpoline-for-tail-call.patch
@@ -0,0 +1,183 @@
+From 8dfc905d7d2e3c68f31eca0178b6137b2e1fc7f9 Mon Sep 17 00:00:00 2001
+From: Daniel Borkmann <daniel@iogearbox.net>
+Date: Thu, 8 Mar 2018 16:17:34 +0100
+Subject: [PATCH 07/14] bpf, x64: implement retpoline for tail call
+
+[ upstream commit a493a87f38cfa48caaa95c9347be2d914c6fdf29 ]
+
+Implement a retpoline [0] for the BPF tail call JIT'ing that converts
+the indirect jump via jmp %rax that is used to make the long jump into
+another JITed BPF image. Since this is subject to speculative execution,
+we need to control the transient instruction sequence here as well
+when CONFIG_RETPOLINE is set, and direct it into a pause + lfence loop.
+The latter aligns also with what gcc / clang emits (e.g. [1]).
+
+JIT dump after patch:
+
+ # bpftool p d x i 1
+ 0: (18) r2 = map[id:1]
+ 2: (b7) r3 = 0
+ 3: (85) call bpf_tail_call#12
+ 4: (b7) r0 = 2
+ 5: (95) exit
+
+With CONFIG_RETPOLINE:
+
+ # bpftool p d j i 1
+ [...]
+ 33: cmp %edx,0x24(%rsi)
+ 36: jbe 0x0000000000000072 |*
+ 38: mov 0x24(%rbp),%eax
+ 3e: cmp $0x20,%eax
+ 41: ja 0x0000000000000072 |
+ 43: add $0x1,%eax
+ 46: mov %eax,0x24(%rbp)
+ 4c: mov 0x90(%rsi,%rdx,8),%rax
+ 54: test %rax,%rax
+ 57: je 0x0000000000000072 |
+ 59: mov 0x28(%rax),%rax
+ 5d: add $0x25,%rax
+ 61: callq 0x000000000000006d |+
+ 66: pause |
+ 68: lfence |
+ 6b: jmp 0x0000000000000066 |
+ 6d: mov %rax,(%rsp) |
+ 71: retq |
+ 72: mov $0x2,%eax
+ [...]
+
+ * relative fall-through jumps in error case
+ + retpoline for indirect jump
+
+Without CONFIG_RETPOLINE:
+
+ # bpftool p d j i 1
+ [...]
+ 33: cmp %edx,0x24(%rsi)
+ 36: jbe 0x0000000000000063 |*
+ 38: mov 0x24(%rbp),%eax
+ 3e: cmp $0x20,%eax
+ 41: ja 0x0000000000000063 |
+ 43: add $0x1,%eax
+ 46: mov %eax,0x24(%rbp)
+ 4c: mov 0x90(%rsi,%rdx,8),%rax
+ 54: test %rax,%rax
+ 57: je 0x0000000000000063 |
+ 59: mov 0x28(%rax),%rax
+ 5d: add $0x25,%rax
+ 61: jmpq *%rax |-
+ 63: mov $0x2,%eax
+ [...]
+
+ * relative fall-through jumps in error case
+ - plain indirect jump as before
+
+ [0] https://support.google.com/faqs/answer/7625886
+ [1] https://github.com/gcc-mirror/gcc/commit/a31e654fa107be968b802786d747e962c2fcdb2b
+
+Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
+Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/include/asm/nospec-branch.h | 37 ++++++++++++++++++++++++++++++++++++
+ arch/x86/net/bpf_jit_comp.c | 9 +++++----
+ 2 files changed, 42 insertions(+), 4 deletions(-)
+
+diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
+index 76b0585..81a1be3 100644
+--- a/arch/x86/include/asm/nospec-branch.h
++++ b/arch/x86/include/asm/nospec-branch.h
+@@ -177,4 +177,41 @@ static inline void indirect_branch_prediction_barrier(void)
+ }
+
+ #endif /* __ASSEMBLY__ */
++
++/*
++ * Below is used in the eBPF JIT compiler and emits the byte sequence
++ * for the following assembly:
++ *
++ * With retpolines configured:
++ *
++ * callq do_rop
++ * spec_trap:
++ * pause
++ * lfence
++ * jmp spec_trap
++ * do_rop:
++ * mov %rax,(%rsp)
++ * retq
++ *
++ * Without retpolines configured:
++ *
++ * jmp *%rax
++ */
++#ifdef CONFIG_RETPOLINE
++# define RETPOLINE_RAX_BPF_JIT_SIZE 17
++# define RETPOLINE_RAX_BPF_JIT() \
++ EMIT1_off32(0xE8, 7); /* callq do_rop */ \
++ /* spec_trap: */ \
++ EMIT2(0xF3, 0x90); /* pause */ \
++ EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \
++ EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \
++ /* do_rop: */ \
++ EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \
++ EMIT1(0xC3); /* retq */
++#else
++# define RETPOLINE_RAX_BPF_JIT_SIZE 2
++# define RETPOLINE_RAX_BPF_JIT() \
++ EMIT2(0xFF, 0xE0); /* jmp *%rax */
++#endif
++
+ #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
+diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
+index 7840331..1f7ed2e 100644
+--- a/arch/x86/net/bpf_jit_comp.c
++++ b/arch/x86/net/bpf_jit_comp.c
+@@ -12,6 +12,7 @@
+ #include <linux/filter.h>
+ #include <linux/if_vlan.h>
+ #include <asm/cacheflush.h>
++#include <asm/nospec-branch.h>
+ #include <linux/bpf.h>
+
+ int bpf_jit_enable __read_mostly;
+@@ -281,7 +282,7 @@ static void emit_bpf_tail_call(u8 **pprog)
+ EMIT2(0x89, 0xD2); /* mov edx, edx */
+ EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */
+ offsetof(struct bpf_array, map.max_entries));
+-#define OFFSET1 43 /* number of bytes to jump */
++#define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* number of bytes to jump */
+ EMIT2(X86_JBE, OFFSET1); /* jbe out */
+ label1 = cnt;
+
+@@ -290,7 +291,7 @@ static void emit_bpf_tail_call(u8 **pprog)
+ */
+ EMIT2_off32(0x8B, 0x85, -STACKSIZE + 36); /* mov eax, dword ptr [rbp - 516] */
+ EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */
+-#define OFFSET2 32
++#define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE)
+ EMIT2(X86_JA, OFFSET2); /* ja out */
+ label2 = cnt;
+ EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */
+@@ -304,7 +305,7 @@ static void emit_bpf_tail_call(u8 **pprog)
+ * goto out;
+ */
+ EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */
+-#define OFFSET3 10
++#define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE)
+ EMIT2(X86_JE, OFFSET3); /* je out */
+ label3 = cnt;
+
+@@ -317,7 +318,7 @@ static void emit_bpf_tail_call(u8 **pprog)
+ * rdi == ctx (1st arg)
+ * rax == prog->bpf_func + prologue_size
+ */
+- EMIT2(0xFF, 0xE0); /* jmp rax */
++ RETPOLINE_RAX_BPF_JIT();
+
+ /* out: */
+ BUILD_BUG_ON(cnt - label1 != OFFSET1);
+--
+2.7.4
+