diff options
Diffstat (limited to 'lib')
79 files changed, 4749 insertions, 973 deletions
diff --git a/lib/842/842_debugfs.h b/lib/842/842_debugfs.h index 277e403e8701..4469407c3e0d 100644 --- a/lib/842/842_debugfs.h +++ b/lib/842/842_debugfs.h @@ -22,8 +22,6 @@ static int __init sw842_debugfs_create(void) return -ENODEV; sw842_debugfs_root = debugfs_create_dir(MODULE_NAME, NULL); - if (IS_ERR(sw842_debugfs_root)) - return PTR_ERR(sw842_debugfs_root); for (i = 0; i < ARRAY_SIZE(template_count); i++) { char name[32]; @@ -46,8 +44,7 @@ static int __init sw842_debugfs_create(void) static void __exit sw842_debugfs_remove(void) { - if (sw842_debugfs_root && !IS_ERR(sw842_debugfs_root)) - debugfs_remove_recursive(sw842_debugfs_root); + debugfs_remove_recursive(sw842_debugfs_root); } #endif diff --git a/lib/Kconfig b/lib/Kconfig index 90623a0e1942..183f92a297ca 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -531,14 +531,6 @@ config LRU_CACHE config CLZ_TAB bool -config DDR - bool "JEDEC DDR data" - help - Data from JEDEC specs for DDR SDRAM memories, - particularly the AC timing parameters and addressing - information. This data is useful for drivers handling - DDR SDRAM controllers. - config IRQ_POLL bool "IRQ polling library" help @@ -562,6 +554,13 @@ config SIGNATURE Digital signature verification. Currently only RSA is supported. Implementation is done using GnuPG MPI library +config DIMLIB + bool + help + Dynamic Interrupt Moderation library. + Implements an algorithm for dynamically changing CQ moderation values + according to run time performance. + # # libfdt files, only selected if needed. # @@ -576,6 +575,11 @@ config OID_REGISTRY config UCS2_STRING tristate +# +# generic vdso +# +source "lib/vdso/Kconfig" + source "lib/fonts/Kconfig" config SG_SPLIT @@ -626,6 +630,9 @@ config SBITMAP config PARMAN tristate "parman" if COMPILE_TEST +config OBJAGG + tristate "objagg" if COMPILE_TEST + config STRING_SELFTEST tristate "Test string functions" @@ -648,6 +655,3 @@ config GENERIC_LIB_CMPDI2 config GENERIC_LIB_UCMPDI2 bool - -config OBJAGG - tristate "objagg" if COMPILE_TEST diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index cbdfae379896..93d97f9b0157 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -277,22 +277,6 @@ config READABLE_ASM to keep kernel developers who have to stare a lot at assembler listings sane. -config UNUSED_SYMBOLS - bool "Enable unused/obsolete exported symbols" - default y if X86 - help - Unused but exported symbols make the kernel needlessly bigger. For - that reason most of these unused exports will soon be removed. This - option is provided temporarily to provide a transition period in case - some external kernel module needs one of these symbols anyway. If you - encounter such a case in your module, consider if you are actually - using the right API. (rationale: since nobody in the kernel is using - this in a module, there is a pretty good chance it's actually the - wrong interface to use). If you really need the symbol, please send a - mail to the linux kernel mailing list mentioning the symbol and why - you really need it, and what the merge plan to the mainline kernel for - your module is. - config DEBUG_FS bool "Debug Filesystem" help @@ -305,22 +289,29 @@ config DEBUG_FS If unsure, say N. -config HEADERS_CHECK - bool "Run 'make headers_check' when building vmlinux" +config HEADERS_INSTALL + bool "Install uapi headers to usr/include" depends on !UML help - This option will extract the user-visible kernel headers whenever - building the kernel, and will run basic sanity checks on them to - ensure that exported files do not attempt to include files which - were not exported, etc. + This option will install uapi headers (headers exported to user-space) + into the usr/include directory for use during the kernel build. + This is unneeded for building the kernel itself, but needed for some + user-space program samples. It is also needed by some features such + as uapi header sanity checks. + +config HEADERS_CHECK + bool "Run sanity checks on uapi headers when building 'all'" + depends on HEADERS_INSTALL + help + This option will run basic sanity checks on uapi headers when + building the 'all' target, for example, ensure that they do not + attempt to include files which were not exported, etc. If you're making modifications to header files which are - relevant for userspace, say 'Y', and check the headers - exported to $(INSTALL_HDR_PATH) (usually 'usr/include' in - your build tree), to make sure they're suitable. + relevant for userspace, say 'Y'. config OPTIMIZE_INLINING - bool "Allow compiler to uninline functions marked 'inline'" + def_bool y help This option determines if the kernel forces gcc to inline the functions developers have marked 'inline'. Doing so takes away freedom from gcc to @@ -331,8 +322,6 @@ config OPTIMIZE_INLINING decision will become the default in the future. Until then this option is there to test gcc for this. - If unsure, say N. - config DEBUG_SECTION_MISMATCH bool "Enable full Section mismatch analysis" help @@ -346,23 +335,13 @@ config DEBUG_SECTION_MISMATCH which results in the code/data being placed in specific sections. The section mismatch analysis is always performed after a full kernel build, and enabling this option causes the following - additional steps to occur: + additional step to occur: - Add the option -fno-inline-functions-called-once to gcc commands. When inlining a function annotated with __init in a non-init function, we would lose the section information and thus the analysis would not catch the illegal reference. This option tells gcc to inline less (but it does result in a larger kernel). - - Run the section mismatch analysis for each module/built-in.a file. - When we run the section mismatch analysis on vmlinux.o, we - lose valuable information about where the mismatch was - introduced. - Running the analysis for each module/built-in.a file - tells where the mismatch happens much closer to the - source. The drawback is that the same mismatch is - reported at least twice. - - Enable verbose reporting from modpost in order to help resolve - the section mismatches that are reported. config SECTION_MISMATCH_WARN_ONLY bool "Make section mismatch errors non-fatal" @@ -595,17 +574,18 @@ config DEBUG_KMEMLEAK In order to access the kmemleak file, debugfs needs to be mounted (usually at /sys/kernel/debug). -config DEBUG_KMEMLEAK_EARLY_LOG_SIZE - int "Maximum kmemleak early log entries" +config DEBUG_KMEMLEAK_MEM_POOL_SIZE + int "Kmemleak memory pool size" depends on DEBUG_KMEMLEAK - range 200 40000 - default 400 + range 200 1000000 + default 16000 help Kmemleak must track all the memory allocations to avoid reporting false positives. Since memory may be allocated or - freed before kmemleak is initialised, an early log buffer is - used to store these actions. If kmemleak reports "early log - buffer exceeded", please increase this value. + freed before kmemleak is fully initialised, use a static pool + of metadata objects to track such callbacks. After kmemleak is + fully initialised, this memory pool acts as an emergency one + if slab allocations fail. config DEBUG_KMEMLEAK_TEST tristate "Simple test for the kernel memory leak detector" @@ -1095,7 +1075,7 @@ config PROVE_LOCKING select DEBUG_SPINLOCK select DEBUG_MUTEXES select DEBUG_RT_MUTEXES if RT_MUTEXES - select DEBUG_RWSEMS if RWSEM_SPIN_ON_OWNER + select DEBUG_RWSEMS select DEBUG_WW_MUTEX_SLOWPATH select DEBUG_LOCK_ALLOC select TRACE_IRQFLAGS @@ -1132,7 +1112,7 @@ config PROVE_LOCKING the proof of observed correctness is also maintained for an arbitrary combination of these separate locking variants. - For more details, see Documentation/locking/lockdep-design.txt. + For more details, see Documentation/locking/lockdep-design.rst. config LOCK_STAT bool "Lock usage statistics" @@ -1146,7 +1126,7 @@ config LOCK_STAT help This feature enables tracking lock contention points - For more details, see Documentation/locking/lockstat.txt + For more details, see Documentation/locking/lockstat.rst This also enables lock events required by "perf lock", subcommand of perf. @@ -1199,10 +1179,10 @@ config DEBUG_WW_MUTEX_SLOWPATH config DEBUG_RWSEMS bool "RW Semaphore debugging: basic checks" - depends on DEBUG_KERNEL && RWSEM_SPIN_ON_OWNER + depends on DEBUG_KERNEL help - This debugging feature allows mismatched rw semaphore locks and unlocks - to be detected and reported. + This debugging feature allows mismatched rw semaphore locks + and unlocks to be detected and reported. config DEBUG_LOCK_ALLOC bool "Lock debugging: detect incorrect freeing of live locks" @@ -1701,7 +1681,7 @@ config LKDTM called lkdtm. Documentation on how to use the module can be found in - Documentation/fault-injection/provoke-crashes.txt + Documentation/fault-injection/provoke-crashes.rst config TEST_LIST_SORT tristate "Linked list sorting test" @@ -1754,6 +1734,18 @@ config RBTREE_TEST A benchmark measuring the performance of the rbtree library. Also includes rbtree invariant checks. +config REED_SOLOMON_TEST + tristate "Reed-Solomon library test" + depends on DEBUG_KERNEL || m + select REED_SOLOMON + select REED_SOLOMON_ENC16 + select REED_SOLOMON_DEC16 + help + This option enables the self-test function of rslib at boot, + or at module load time. + + If unsure, say N. + config INTERVAL_TREE_TEST tristate "Interval tree test" depends on DEBUG_KERNEL @@ -1858,6 +1850,14 @@ config TEST_PARMAN If unsure, say N. +config TEST_IRQ_TIMINGS + bool "IRQ timings selftest" + depends on IRQ_TIMINGS + help + Enable this option to test the irq timings code on boot. + + If unsure, say N. + config TEST_LKM tristate "Test module loading with 'hello world' module" depends on m @@ -1909,6 +1909,15 @@ config TEST_BPF If unsure, say N. +config TEST_BLACKHOLE_DEV + tristate "Test blackhole netdev functionality" + depends on m && NET + help + This builds the "test_blackhole_dev" module that validates the + data path through this blackhole netdev. + + If unsure, say N. + config FIND_BIT_BENCHMARK tristate "Test find_bit functions" help @@ -2040,6 +2049,14 @@ config TEST_STACKINIT If unsure, say N. +config TEST_MEMINIT + tristate "Test heap/page initialization" + help + Test if the kernel is zero-initializing heap and page allocations. + This can be useful to test init_on_alloc and init_on_free features. + + If unsure, say N. + endif # RUNTIME_TESTING_MENU config MEMTEST diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index 4fafba1a923b..6c9682ce0254 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -106,7 +106,6 @@ endchoice config KASAN_STACK_ENABLE bool "Enable stack instrumentation (unsafe)" if CC_IS_CLANG && !COMPILE_TEST - default !(CLANG_VERSION < 90000) depends on KASAN help The LLVM stack address sanitizer has a know problem that @@ -115,11 +114,11 @@ config KASAN_STACK_ENABLE Disabling asan-stack makes it safe to run kernels build with clang-8 with KASAN enabled, though it loses some of the functionality. - This feature is always disabled when compile-testing with clang-8 - or earlier to avoid cluttering the output in stack overflow - warnings, but clang-8 users can still enable it for builds without - CONFIG_COMPILE_TEST. On gcc and later clang versions it is - assumed to always be safe to use and enabled by default. + This feature is always disabled when compile-testing with clang + to avoid cluttering the output in stack overflow warnings, + but clang users can still enable it for builds without + CONFIG_COMPILE_TEST. On gcc it is assumed to always be safe + to use and enabled by default. config KASAN_STACK int @@ -135,6 +134,14 @@ config KASAN_S390_4_LEVEL_PAGING to 3TB of RAM with KASan enabled). This options allows to force 4-level paging instead. +config KASAN_SW_TAGS_IDENTIFY + bool "Enable memory corruption identification" + depends on KASAN_SW_TAGS + help + This option enables best-effort identification of bug type + (use-after-free or out-of-bounds) at the cost of increased + memory consumption. + config TEST_KASAN tristate "Module for testing KASAN for bug detection" depends on m && KASAN diff --git a/lib/Makefile b/lib/Makefile index fb7697031a79..c5892807e06f 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -21,10 +21,6 @@ KCOV_INSTRUMENT_dynamic_debug.o := n ifdef CONFIG_AMD_MEM_ENCRYPT KASAN_SANITIZE_string.o := n -ifdef CONFIG_FUNCTION_TRACER -CFLAGS_REMOVE_string.o = -pg -endif - CFLAGS_string.o := $(call cc-option, -fno-stack-protector) endif @@ -91,6 +87,8 @@ obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o obj-$(CONFIG_TEST_MEMCAT_P) += test_memcat_p.o obj-$(CONFIG_TEST_OBJAGG) += test_objagg.o obj-$(CONFIG_TEST_STACKINIT) += test_stackinit.o +obj-$(CONFIG_TEST_BLACKHOLE_DEV) += test_blackhole_dev.o +obj-$(CONFIG_TEST_MEMINIT) += test_meminit.o obj-$(CONFIG_TEST_LIVEPATCH) += livepatch/ @@ -102,7 +100,7 @@ endif obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any) -obj-y += math/ +obj-y += math/ crypto/ obj-$(CONFIG_GENERIC_IOMAP) += iomap.o obj-$(CONFIG_GENERIC_PCI_IOMAP) += pci_iomap.o @@ -202,12 +200,11 @@ obj-$(CONFIG_GLOB) += glob.o obj-$(CONFIG_GLOB_SELFTEST) += globtest.o obj-$(CONFIG_MPILIB) += mpi/ +obj-$(CONFIG_DIMLIB) += dim/ obj-$(CONFIG_SIGNATURE) += digsig.o lib-$(CONFIG_CLZ_TAB) += clz_tab.o -obj-$(CONFIG_DDR) += jedec_ddr_data.o - obj-$(CONFIG_GENERIC_STRNCPY_FROM_USER) += strncpy_from_user.o obj-$(CONFIG_GENERIC_STRNLEN_USER) += strnlen_user.o @@ -278,7 +275,8 @@ obj-$(CONFIG_UCS2_STRING) += ucs2_string.o obj-$(CONFIG_UBSAN) += ubsan.o UBSAN_SANITIZE_ubsan.o := n -CFLAGS_ubsan.o := $(call cc-option, -fno-conserve-stack -fno-stack-protector) +KASAN_SANITIZE_ubsan.o := n +CFLAGS_ubsan.o := $(call cc-option, -fno-stack-protector) $(DISABLE_STACKLEAK_PLUGIN) obj-$(CONFIG_SBITMAP) += sbitmap.o diff --git a/lib/atomic64.c b/lib/atomic64.c index 7e6905751522..e98c85a99787 100644 --- a/lib/atomic64.c +++ b/lib/atomic64.c @@ -42,11 +42,11 @@ static inline raw_spinlock_t *lock_addr(const atomic64_t *v) return &atomic64_lock[addr & (NR_LOCKS - 1)].lock; } -long long atomic64_read(const atomic64_t *v) +s64 atomic64_read(const atomic64_t *v) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); - long long val; + s64 val; raw_spin_lock_irqsave(lock, flags); val = v->counter; @@ -55,7 +55,7 @@ long long atomic64_read(const atomic64_t *v) } EXPORT_SYMBOL(atomic64_read); -void atomic64_set(atomic64_t *v, long long i) +void atomic64_set(atomic64_t *v, s64 i) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -67,7 +67,7 @@ void atomic64_set(atomic64_t *v, long long i) EXPORT_SYMBOL(atomic64_set); #define ATOMIC64_OP(op, c_op) \ -void atomic64_##op(long long a, atomic64_t *v) \ +void atomic64_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ @@ -79,11 +79,11 @@ void atomic64_##op(long long a, atomic64_t *v) \ EXPORT_SYMBOL(atomic64_##op); #define ATOMIC64_OP_RETURN(op, c_op) \ -long long atomic64_##op##_return(long long a, atomic64_t *v) \ +s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ - long long val; \ + s64 val; \ \ raw_spin_lock_irqsave(lock, flags); \ val = (v->counter c_op a); \ @@ -93,11 +93,11 @@ long long atomic64_##op##_return(long long a, atomic64_t *v) \ EXPORT_SYMBOL(atomic64_##op##_return); #define ATOMIC64_FETCH_OP(op, c_op) \ -long long atomic64_fetch_##op(long long a, atomic64_t *v) \ +s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ - long long val; \ + s64 val; \ \ raw_spin_lock_irqsave(lock, flags); \ val = v->counter; \ @@ -130,11 +130,11 @@ ATOMIC64_OPS(xor, ^=) #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -long long atomic64_dec_if_positive(atomic64_t *v) +s64 atomic64_dec_if_positive(atomic64_t *v) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); - long long val; + s64 val; raw_spin_lock_irqsave(lock, flags); val = v->counter - 1; @@ -145,11 +145,11 @@ long long atomic64_dec_if_positive(atomic64_t *v) } EXPORT_SYMBOL(atomic64_dec_if_positive); -long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n) +s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); - long long val; + s64 val; raw_spin_lock_irqsave(lock, flags); val = v->counter; @@ -160,11 +160,11 @@ long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n) } EXPORT_SYMBOL(atomic64_cmpxchg); -long long atomic64_xchg(atomic64_t *v, long long new) +s64 atomic64_xchg(atomic64_t *v, s64 new) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); - long long val; + s64 val; raw_spin_lock_irqsave(lock, flags); val = v->counter; @@ -174,11 +174,11 @@ long long atomic64_xchg(atomic64_t *v, long long new) } EXPORT_SYMBOL(atomic64_xchg); -long long atomic64_fetch_add_unless(atomic64_t *v, long long a, long long u) +s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); - long long val; + s64 val; raw_spin_lock_irqsave(lock, flags); val = v->counter; diff --git a/lib/bitmap.c b/lib/bitmap.c index bbe2589e8497..f9e834841e94 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -59,6 +59,26 @@ int __bitmap_equal(const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_equal); +bool __bitmap_or_equal(const unsigned long *bitmap1, + const unsigned long *bitmap2, + const unsigned long *bitmap3, + unsigned int bits) +{ + unsigned int k, lim = bits / BITS_PER_LONG; + unsigned long tmp; + + for (k = 0; k < lim; ++k) { + if ((bitmap1[k] | bitmap2[k]) != bitmap3[k]) + return false; + } + + if (!(bits % BITS_PER_LONG)) + return true; + + tmp = (bitmap1[k] | bitmap2[k]) ^ bitmap3[k]; + return (tmp & BITMAP_LAST_WORD_MASK(bits)) == 0; +} + void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int bits) { unsigned int k, lim = BITS_TO_LONGS(bits); diff --git a/lib/bug.c b/lib/bug.c index 1077366f496b..8c98af0bf585 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -181,6 +181,15 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) } } + /* + * BUG() and WARN_ON() families don't print a custom debug message + * before triggering the exception handler, so we must add the + * "cut here" line now. WARN() issues its own "cut here" before the + * extra debugging message it writes before triggering the handler. + */ + if ((bug->flags & BUGFLAG_NO_CUT_HERE) == 0) + printk(KERN_DEFAULT CUT_HERE); + if (warning) { /* this is a WARN_ON rather than BUG/BUG_ON */ __warn(file, line, (void *)bugaddr, BUG_GET_TAINT(bug), regs, @@ -188,8 +197,6 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) return BUG_TRAP_TYPE_WARN; } - printk(KERN_DEFAULT CUT_HERE); - if (file) pr_crit("kernel BUG at %s:%u!\n", file, line); else diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile new file mode 100644 index 000000000000..cbe0b6a6450d --- /dev/null +++ b/lib/crypto/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_CRYPTO_LIB_AES) += libaes.o +libaes-y := aes.o + +obj-$(CONFIG_CRYPTO_LIB_ARC4) += libarc4.o +libarc4-y := arc4.o + +obj-$(CONFIG_CRYPTO_LIB_DES) += libdes.o +libdes-y := des.o + +obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o +libsha256-y := sha256.o diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c new file mode 100644 index 000000000000..827fe89922ff --- /dev/null +++ b/lib/crypto/aes.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org> + */ + +#include <crypto/aes.h> +#include <linux/crypto.h> +#include <linux/module.h> +#include <asm/unaligned.h> + +/* + * Emit the sbox as volatile const to prevent the compiler from doing + * constant folding on sbox references involving fixed indexes. + */ +static volatile const u8 __cacheline_aligned aes_sbox[] = { + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, + 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, + 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, + 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, + 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, + 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, + 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, + 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, + 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, + 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, + 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, + 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, + 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, + 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, + 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, + 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, + 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, +}; + +static volatile const u8 __cacheline_aligned aes_inv_sbox[] = { + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, + 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, + 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, + 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, + 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, + 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, + 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, + 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, + 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, + 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, + 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, + 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, + 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, + 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, + 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, + 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, + 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d, +}; + +extern const u8 crypto_aes_sbox[256] __alias(aes_sbox); +extern const u8 crypto_aes_inv_sbox[256] __alias(aes_inv_sbox); + +EXPORT_SYMBOL(crypto_aes_sbox); +EXPORT_SYMBOL(crypto_aes_inv_sbox); + +static u32 mul_by_x(u32 w) +{ + u32 x = w & 0x7f7f7f7f; + u32 y = w & 0x80808080; + + /* multiply by polynomial 'x' (0b10) in GF(2^8) */ + return (x << 1) ^ (y >> 7) * 0x1b; +} + +static u32 mul_by_x2(u32 w) +{ + u32 x = w & 0x3f3f3f3f; + u32 y = w & 0x80808080; + u32 z = w & 0x40404040; + + /* multiply by polynomial 'x^2' (0b100) in GF(2^8) */ + return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b; +} + +static u32 mix_columns(u32 x) +{ + /* + * Perform the following matrix multiplication in GF(2^8) + * + * | 0x2 0x3 0x1 0x1 | | x[0] | + * | 0x1 0x2 0x3 0x1 | | x[1] | + * | 0x1 0x1 0x2 0x3 | x | x[2] | + * | 0x3 0x1 0x1 0x2 | | x[3] | + */ + u32 y = mul_by_x(x) ^ ror32(x, 16); + + return y ^ ror32(x ^ y, 8); +} + +static u32 inv_mix_columns(u32 x) +{ + /* + * Perform the following matrix multiplication in GF(2^8) + * + * | 0xe 0xb 0xd 0x9 | | x[0] | + * | 0x9 0xe 0xb 0xd | | x[1] | + * | 0xd 0x9 0xe 0xb | x | x[2] | + * | 0xb 0xd 0x9 0xe | | x[3] | + * + * which can conveniently be reduced to + * + * | 0x2 0x3 0x1 0x1 | | 0x5 0x0 0x4 0x0 | | x[0] | + * | 0x1 0x2 0x3 0x1 | | 0x0 0x5 0x0 0x4 | | x[1] | + * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] | + * | 0x3 0x1 0x1 0x2 | | 0x0 0x4 0x0 0x5 | | x[3] | + */ + u32 y = mul_by_x2(x); + + return mix_columns(x ^ y ^ ror32(y, 16)); +} + +static __always_inline u32 subshift(u32 in[], int pos) +{ + return (aes_sbox[in[pos] & 0xff]) ^ + (aes_sbox[(in[(pos + 1) % 4] >> 8) & 0xff] << 8) ^ + (aes_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^ + (aes_sbox[(in[(pos + 3) % 4] >> 24) & 0xff] << 24); +} + +static __always_inline u32 inv_subshift(u32 in[], int pos) +{ + return (aes_inv_sbox[in[pos] & 0xff]) ^ + (aes_inv_sbox[(in[(pos + 3) % 4] >> 8) & 0xff] << 8) ^ + (aes_inv_sbox[(in[(pos + 2) % 4] >> 16) & 0xff] << 16) ^ + (aes_inv_sbox[(in[(pos + 1) % 4] >> 24) & 0xff] << 24); +} + +static u32 subw(u32 in) +{ + return (aes_sbox[in & 0xff]) ^ + (aes_sbox[(in >> 8) & 0xff] << 8) ^ + (aes_sbox[(in >> 16) & 0xff] << 16) ^ + (aes_sbox[(in >> 24) & 0xff] << 24); +} + +/** + * aes_expandkey - Expands the AES key as described in FIPS-197 + * @ctx: The location where the computed key will be stored. + * @in_key: The supplied key. + * @key_len: The length of the supplied key. + * + * Returns 0 on success. The function fails only if an invalid key size (or + * pointer) is supplied. + * The expanded key size is 240 bytes (max of 14 rounds with a unique 16 bytes + * key schedule plus a 16 bytes key which is used before the first round). + * The decryption key is prepared for the "Equivalent Inverse Cipher" as + * described in FIPS-197. The first slot (16 bytes) of each key (enc or dec) is + * for the initial combination, the second slot for the first round and so on. + */ +int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, + unsigned int key_len) +{ + u32 kwords = key_len / sizeof(u32); + u32 rc, i, j; + int err; + + err = aes_check_keylen(key_len); + if (err) + return err; + + ctx->key_length = key_len; + + for (i = 0; i < kwords; i++) + ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32)); + + for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) { + u32 *rki = ctx->key_enc + (i * kwords); + u32 *rko = rki + kwords; + + rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0]; + rko[1] = rko[0] ^ rki[1]; + rko[2] = rko[1] ^ rki[2]; + rko[3] = rko[2] ^ rki[3]; + + if (key_len == AES_KEYSIZE_192) { + if (i >= 7) + break; + rko[4] = rko[3] ^ rki[4]; + rko[5] = rko[4] ^ rki[5]; + } else if (key_len == AES_KEYSIZE_256) { + if (i >= 6) + break; + rko[4] = subw(rko[3]) ^ rki[4]; + rko[5] = rko[4] ^ rki[5]; + rko[6] = rko[5] ^ rki[6]; + rko[7] = rko[6] ^ rki[7]; + } + } + + /* + * Generate the decryption keys for the Equivalent Inverse Cipher. + * This involves reversing the order of the round keys, and applying + * the Inverse Mix Columns transformation to all but the first and + * the last one. + */ + ctx->key_dec[0] = ctx->key_enc[key_len + 24]; + ctx->key_dec[1] = ctx->key_enc[key_len + 25]; + ctx->key_dec[2] = ctx->key_enc[key_len + 26]; + ctx->key_dec[3] = ctx->key_enc[key_len + 27]; + + for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) { + ctx->key_dec[i] = inv_mix_columns(ctx->key_enc[j]); + ctx->key_dec[i + 1] = inv_mix_columns(ctx->key_enc[j + 1]); + ctx->key_dec[i + 2] = inv_mix_columns(ctx->key_enc[j + 2]); + ctx->key_dec[i + 3] = inv_mix_columns(ctx->key_enc[j + 3]); + } + + ctx->key_dec[i] = ctx->key_enc[0]; + ctx->key_dec[i + 1] = ctx->key_enc[1]; + ctx->key_dec[i + 2] = ctx->key_enc[2]; + ctx->key_dec[i + 3] = ctx->key_enc[3]; + + return 0; +} +EXPORT_SYMBOL(aes_expandkey); + +/** + * aes_encrypt - Encrypt a single AES block + * @ctx: Context struct containing the key schedule + * @out: Buffer to store the ciphertext + * @in: Buffer containing the plaintext + */ +void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in) +{ + const u32 *rkp = ctx->key_enc + 4; + int rounds = 6 + ctx->key_length / 4; + u32 st0[4], st1[4]; + int round; + + st0[0] = ctx->key_enc[0] ^ get_unaligned_le32(in); + st0[1] = ctx->key_enc[1] ^ get_unaligned_le32(in + 4); + st0[2] = ctx->key_enc[2] ^ get_unaligned_le32(in + 8); + st0[3] = ctx->key_enc[3] ^ get_unaligned_le32(in + 12); + + /* + * Force the compiler to emit data independent Sbox references, + * by xoring the input with Sbox values that are known to add up + * to zero. This pulls the entire Sbox into the D-cache before any + * data dependent lookups are done. + */ + st0[0] ^= aes_sbox[ 0] ^ aes_sbox[ 64] ^ aes_sbox[134] ^ aes_sbox[195]; + st0[1] ^= aes_sbox[16] ^ aes_sbox[ 82] ^ aes_sbox[158] ^ aes_sbox[221]; + st0[2] ^= aes_sbox[32] ^ aes_sbox[ 96] ^ aes_sbox[160] ^ aes_sbox[234]; + st0[3] ^= aes_sbox[48] ^ aes_sbox[112] ^ aes_sbox[186] ^ aes_sbox[241]; + + for (round = 0;; round += 2, rkp += 8) { + st1[0] = mix_columns(subshift(st0, 0)) ^ rkp[0]; + st1[1] = mix_columns(subshift(st0, 1)) ^ rkp[1]; + st1[2] = mix_columns(subshift(st0, 2)) ^ rkp[2]; + st1[3] = mix_columns(subshift(st0, 3)) ^ rkp[3]; + + if (round == rounds - 2) + break; + + st0[0] = mix_columns(subshift(st1, 0)) ^ rkp[4]; + st0[1] = mix_columns(subshift(st1, 1)) ^ rkp[5]; + st0[2] = mix_columns(subshift(st1, 2)) ^ rkp[6]; + st0[3] = mix_columns(subshift(st1, 3)) ^ rkp[7]; + } + + put_unaligned_le32(subshift(st1, 0) ^ rkp[4], out); + put_unaligned_le32(subshift(st1, 1) ^ rkp[5], out + 4); + put_unaligned_le32(subshift(st1, 2) ^ rkp[6], out + 8); + put_unaligned_le32(subshift(st1, 3) ^ rkp[7], out + 12); +} +EXPORT_SYMBOL(aes_encrypt); + +/** + * aes_decrypt - Decrypt a single AES block + * @ctx: Context struct containing the key schedule + * @out: Buffer to store the plaintext + * @in: Buffer containing the ciphertext + */ +void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in) +{ + const u32 *rkp = ctx->key_dec + 4; + int rounds = 6 + ctx->key_length / 4; + u32 st0[4], st1[4]; + int round; + + st0[0] = ctx->key_dec[0] ^ get_unaligned_le32(in); + st0[1] = ctx->key_dec[1] ^ get_unaligned_le32(in + 4); + st0[2] = ctx->key_dec[2] ^ get_unaligned_le32(in + 8); + st0[3] = ctx->key_dec[3] ^ get_unaligned_le32(in + 12); + + /* + * Force the compiler to emit data independent Sbox references, + * by xoring the input with Sbox values that are known to add up + * to zero. This pulls the entire Sbox into the D-cache before any + * data dependent lookups are done. + */ + st0[0] ^= aes_inv_sbox[ 0] ^ aes_inv_sbox[ 64] ^ aes_inv_sbox[129] ^ aes_inv_sbox[200]; + st0[1] ^= aes_inv_sbox[16] ^ aes_inv_sbox[ 83] ^ aes_inv_sbox[150] ^ aes_inv_sbox[212]; + st0[2] ^= aes_inv_sbox[32] ^ aes_inv_sbox[ 96] ^ aes_inv_sbox[160] ^ aes_inv_sbox[236]; + st0[3] ^= aes_inv_sbox[48] ^ aes_inv_sbox[112] ^ aes_inv_sbox[187] ^ aes_inv_sbox[247]; + + for (round = 0;; round += 2, rkp += 8) { + st1[0] = inv_mix_columns(inv_subshift(st0, 0)) ^ rkp[0]; + st1[1] = inv_mix_columns(inv_subshift(st0, 1)) ^ rkp[1]; + st1[2] = inv_mix_columns(inv_subshift(st0, 2)) ^ rkp[2]; + st1[3] = inv_mix_columns(inv_subshift(st0, 3)) ^ rkp[3]; + + if (round == rounds - 2) + break; + + st0[0] = inv_mix_columns(inv_subshift(st1, 0)) ^ rkp[4]; + st0[1] = inv_mix_columns(inv_subshift(st1, 1)) ^ rkp[5]; + st0[2] = inv_mix_columns(inv_subshift(st1, 2)) ^ rkp[6]; + st0[3] = inv_mix_columns(inv_subshift(st1, 3)) ^ rkp[7]; + } + + put_unaligned_le32(inv_subshift(st1, 0) ^ rkp[4], out); + put_unaligned_le32(inv_subshift(st1, 1) ^ rkp[5], out + 4); + put_unaligned_le32(inv_subshift(st1, 2) ^ rkp[6], out + 8); + put_unaligned_le32(inv_subshift(st1, 3) ^ rkp[7], out + 12); +} +EXPORT_SYMBOL(aes_decrypt); + +MODULE_DESCRIPTION("Generic AES library"); +MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); +MODULE_LICENSE("GPL v2"); diff --git a/lib/crypto/arc4.c b/lib/crypto/arc4.c new file mode 100644 index 000000000000..c2020f19c652 --- /dev/null +++ b/lib/crypto/arc4.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Cryptographic API + * + * ARC4 Cipher Algorithm + * + * Jon Oberheide <jon@oberheide.org> + */ + +#include <crypto/arc4.h> +#include <linux/module.h> + +int arc4_setkey(struct arc4_ctx *ctx, const u8 *in_key, unsigned int key_len) +{ + int i, j = 0, k = 0; + + ctx->x = 1; + ctx->y = 0; + + for (i = 0; i < 256; i++) + ctx->S[i] = i; + + for (i = 0; i < 256; i++) { + u32 a = ctx->S[i]; + + j = (j + in_key[k] + a) & 0xff; + ctx->S[i] = ctx->S[j]; + ctx->S[j] = a; + if (++k >= key_len) + k = 0; + } + + return 0; +} +EXPORT_SYMBOL(arc4_setkey); + +void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, unsigned int len) +{ + u32 *const S = ctx->S; + u32 x, y, a, b; + u32 ty, ta, tb; + + if (len == 0) + return; + + x = ctx->x; + y = ctx->y; + + a = S[x]; + y = (y + a) & 0xff; + b = S[y]; + + do { + S[y] = a; + a = (a + b) & 0xff; + S[x] = b; + x = (x + 1) & 0xff; + ta = S[x]; + ty = (y + ta) & 0xff; + tb = S[ty]; + *out++ = *in++ ^ S[a]; + if (--len == 0) + break; + y = ty; + a = ta; + b = tb; + } while (true); + + ctx->x = x; + ctx->y = y; +} +EXPORT_SYMBOL(arc4_crypt); + +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/des.c b/lib/crypto/des.c new file mode 100644 index 000000000000..ef5bb8822aba --- /dev/null +++ b/lib/crypto/des.c @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Cryptographic API. + * + * DES & Triple DES EDE Cipher Algorithms. + * + * Copyright (c) 2005 Dag Arne Osvik <da@osvik.no> + */ + +#include <linux/bitops.h> +#include <linux/compiler.h> +#include <linux/crypto.h> +#include <linux/errno.h> +#include <linux/fips.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/string.h> +#include <linux/types.h> + +#include <asm/unaligned.h> + +#include <crypto/des.h> +#include <crypto/internal/des.h> + +#define ROL(x, r) ((x) = rol32((x), (r))) +#define ROR(x, r) ((x) = ror32((x), (r))) + +/* Lookup tables for key expansion */ + +static const u8 pc1[256] = { + 0x00, 0x00, 0x40, 0x04, 0x10, 0x10, 0x50, 0x14, + 0x04, 0x40, 0x44, 0x44, 0x14, 0x50, 0x54, 0x54, + 0x02, 0x02, 0x42, 0x06, 0x12, 0x12, 0x52, 0x16, + 0x06, 0x42, 0x46, 0x46, 0x16, 0x52, 0x56, 0x56, + 0x80, 0x08, 0xc0, 0x0c, 0x90, 0x18, 0xd0, 0x1c, + 0x84, 0x48, 0xc4, 0x4c, 0x94, 0x58, 0xd4, 0x5c, + 0x82, 0x0a, 0xc2, 0x0e, 0x92, 0x1a, 0xd2, 0x1e, + 0x86, 0x4a, 0xc6, 0x4e, 0x96, 0x5a, 0xd6, 0x5e, + 0x20, 0x20, 0x60, 0x24, 0x30, 0x30, 0x70, 0x34, + 0x24, 0x60, 0x64, 0x64, 0x34, 0x70, 0x74, 0x74, + 0x22, 0x22, 0x62, 0x26, 0x32, 0x32, 0x72, 0x36, + 0x26, 0x62, 0x66, 0x66, 0x36, 0x72, 0x76, 0x76, + 0xa0, 0x28, 0xe0, 0x2c, 0xb0, 0x38, 0xf0, 0x3c, + 0xa4, 0x68, 0xe4, 0x6c, 0xb4, 0x78, 0xf4, 0x7c, + 0xa2, 0x2a, 0xe2, 0x2e, 0xb2, 0x3a, 0xf2, 0x3e, + 0xa6, 0x6a, 0xe6, 0x6e, 0xb6, 0x7a, 0xf6, 0x7e, + 0x08, 0x80, 0x48, 0x84, 0x18, 0x90, 0x58, 0x94, + 0x0c, 0xc0, 0x4c, 0xc4, 0x1c, 0xd0, 0x5c, 0xd4, + 0x0a, 0x82, 0x4a, 0x86, 0x1a, 0x92, 0x5a, 0x96, + 0x0e, 0xc2, 0x4e, 0xc6, 0x1e, 0xd2, 0x5e, 0xd6, + 0x88, 0x88, 0xc8, 0x8c, 0x98, 0x98, 0xd8, 0x9c, + 0x8c, 0xc8, 0xcc, 0xcc, 0x9c, 0xd8, 0xdc, 0xdc, + 0x8a, 0x8a, 0xca, 0x8e, 0x9a, 0x9a, 0xda, 0x9e, + 0x8e, 0xca, 0xce, 0xce, 0x9e, 0xda, 0xde, 0xde, + 0x28, 0xa0, 0x68, 0xa4, 0x38, 0xb0, 0x78, 0xb4, + 0x2c, 0xe0, 0x6c, 0xe4, 0x3c, 0xf0, 0x7c, 0xf4, + 0x2a, 0xa2, 0x6a, 0xa6, 0x3a, 0xb2, 0x7a, 0xb6, + 0x2e, 0xe2, 0x6e, 0xe6, 0x3e, 0xf2, 0x7e, 0xf6, + 0xa8, 0xa8, 0xe8, 0xac, 0xb8, 0xb8, 0xf8, 0xbc, + 0xac, 0xe8, 0xec, 0xec, 0xbc, 0xf8, 0xfc, 0xfc, + 0xaa, 0xaa, 0xea, 0xae, 0xba, 0xba, 0xfa, 0xbe, + 0xae, 0xea, 0xee, 0xee, 0xbe, 0xfa, 0xfe, 0xfe +}; + +static const u8 rs[256] = { + 0x00, 0x00, 0x80, 0x80, 0x02, 0x02, 0x82, 0x82, + 0x04, 0x04, 0x84, 0x84, 0x06, 0x06, 0x86, 0x86, + 0x08, 0x08, 0x88, 0x88, 0x0a, 0x0a, 0x8a, 0x8a, + 0x0c, 0x0c, 0x8c, 0x8c, 0x0e, 0x0e, 0x8e, 0x8e, + 0x10, 0x10, 0x90, 0x90, 0x12, 0x12, 0x92, 0x92, + 0x14, 0x14, 0x94, 0x94, 0x16, 0x16, 0x96, 0x96, + 0x18, 0x18, 0x98, 0x98, 0x1a, 0x1a, 0x9a, 0x9a, + 0x1c, 0x1c, 0x9c, 0x9c, 0x1e, 0x1e, 0x9e, 0x9e, + 0x20, 0x20, 0xa0, 0xa0, 0x22, 0x22, 0xa2, 0xa2, + 0x24, 0x24, 0xa4, 0xa4, 0x26, 0x26, 0xa6, 0xa6, + 0x28, 0x28, 0xa8, 0xa8, 0x2a, 0x2a, 0xaa, 0xaa, + 0x2c, 0x2c, 0xac, 0xac, 0x2e, 0x2e, 0xae, 0xae, + 0x30, 0x30, 0xb0, 0xb0, 0x32, 0x32, 0xb2, 0xb2, + 0x34, 0x34, 0xb4, 0xb4, 0x36, 0x36, 0xb6, 0xb6, + 0x38, 0x38, 0xb8, 0xb8, 0x3a, 0x3a, 0xba, 0xba, + 0x3c, 0x3c, 0xbc, 0xbc, 0x3e, 0x3e, 0xbe, 0xbe, + 0x40, 0x40, 0xc0, 0xc0, 0x42, 0x42, 0xc2, 0xc2, + 0x44, 0x44, 0xc4, 0xc4, 0x46, 0x46, 0xc6, 0xc6, + 0x48, 0x48, 0xc8, 0xc8, 0x4a, 0x4a, 0xca, 0xca, + 0x4c, 0x4c, 0xcc, 0xcc, 0x4e, 0x4e, 0xce, 0xce, + 0x50, 0x50, 0xd0, 0xd0, 0x52, 0x52, 0xd2, 0xd2, + 0x54, 0x54, 0xd4, 0xd4, 0x56, 0x56, 0xd6, 0xd6, + 0x58, 0x58, 0xd8, 0xd8, 0x5a, 0x5a, 0xda, 0xda, + 0x5c, 0x5c, 0xdc, 0xdc, 0x5e, 0x5e, 0xde, 0xde, + 0x60, 0x60, 0xe0, 0xe0, 0x62, 0x62, 0xe2, 0xe2, + 0x64, 0x64, 0xe4, 0xe4, 0x66, 0x66, 0xe6, 0xe6, + 0x68, 0x68, 0xe8, 0xe8, 0x6a, 0x6a, 0xea, 0xea, + 0x6c, 0x6c, 0xec, 0xec, 0x6e, 0x6e, 0xee, 0xee, + 0x70, 0x70, 0xf0, 0xf0, 0x72, 0x72, 0xf2, 0xf2, + 0x74, 0x74, 0xf4, 0xf4, 0x76, 0x76, 0xf6, 0xf6, + 0x78, 0x78, 0xf8, 0xf8, 0x7a, 0x7a, 0xfa, 0xfa, + 0x7c, 0x7c, 0xfc, 0xfc, 0x7e, 0x7e, 0xfe, 0xfe +}; + +static const u32 pc2[1024] = { + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00040000, 0x00000000, 0x04000000, 0x00100000, + 0x00400000, 0x00000008, 0x00000800, 0x40000000, + 0x00440000, 0x00000008, 0x04000800, 0x40100000, + 0x00000400, 0x00000020, 0x08000000, 0x00000100, + 0x00040400, 0x00000020, 0x0c000000, 0x00100100, + 0x00400400, 0x00000028, 0x08000800, 0x40000100, + 0x00440400, 0x00000028, 0x0c000800, 0x40100100, + 0x80000000, 0x00000010, 0x00000000, 0x00800000, + 0x80040000, 0x00000010, 0x04000000, 0x00900000, + 0x80400000, 0x00000018, 0x00000800, 0x40800000, + 0x80440000, 0x00000018, 0x04000800, 0x40900000, + 0x80000400, 0x00000030, 0x08000000, 0x00800100, + 0x80040400, 0x00000030, 0x0c000000, 0x00900100, + 0x80400400, 0x00000038, 0x08000800, 0x40800100, + 0x80440400, 0x00000038, 0x0c000800, 0x40900100, + 0x10000000, 0x00000000, 0x00200000, 0x00001000, + 0x10040000, 0x00000000, 0x04200000, 0x00101000, + 0x10400000, 0x00000008, 0x00200800, 0x40001000, + 0x10440000, 0x00000008, 0x04200800, 0x40101000, + 0x10000400, 0x00000020, 0x08200000, 0x00001100, + 0x10040400, 0x00000020, 0x0c200000, 0x00101100, + 0x10400400, 0x00000028, 0x08200800, 0x40001100, + 0x10440400, 0x00000028, 0x0c200800, 0x40101100, + 0x90000000, 0x00000010, 0x00200000, 0x00801000, + 0x90040000, 0x00000010, 0x04200000, 0x00901000, + 0x90400000, 0x00000018, 0x00200800, 0x40801000, + 0x90440000, 0x00000018, 0x04200800, 0x40901000, + 0x90000400, 0x00000030, 0x08200000, 0x00801100, + 0x90040400, 0x00000030, 0x0c200000, 0x00901100, + 0x90400400, 0x00000038, 0x08200800, 0x40801100, + 0x90440400, 0x00000038, 0x0c200800, 0x40901100, + 0x00000200, 0x00080000, 0x00000000, 0x00000004, + 0x00040200, 0x00080000, 0x04000000, 0x00100004, + 0x00400200, 0x00080008, 0x00000800, 0x40000004, + 0x00440200, 0x00080008, 0x04000800, 0x40100004, + 0x00000600, 0x00080020, 0x08000000, 0x00000104, + 0x00040600, 0x00080020, 0x0c000000, 0x00100104, + 0x00400600, 0x00080028, 0x08000800, 0x40000104, + 0x00440600, 0x00080028, 0x0c000800, 0x40100104, + 0x80000200, 0x00080010, 0x00000000, 0x00800004, + 0x80040200, 0x00080010, 0x04000000, 0x00900004, + 0x80400200, 0x00080018, 0x00000800, 0x40800004, + 0x80440200, 0x00080018, 0x04000800, 0x40900004, + 0x80000600, 0x00080030, 0x08000000, 0x00800104, + 0x80040600, 0x00080030, 0x0c000000, 0x00900104, + 0x80400600, 0x00080038, 0x08000800, 0x40800104, + 0x80440600, 0x00080038, 0x0c000800, 0x40900104, + 0x10000200, 0x00080000, 0x00200000, 0x00001004, + 0x10040200, 0x00080000, 0x04200000, 0x00101004, + 0x10400200, 0x00080008, 0x00200800, 0x40001004, + 0x10440200, 0x00080008, 0x04200800, 0x40101004, + 0x10000600, 0x00080020, 0x08200000, 0x00001104, + 0x10040600, 0x00080020, 0x0c200000, 0x00101104, + 0x10400600, 0x00080028, 0x08200800, 0x40001104, + 0x10440600, 0x00080028, 0x0c200800, 0x40101104, + 0x90000200, 0x00080010, 0x00200000, 0x00801004, + 0x90040200, 0x00080010, 0x04200000, 0x00901004, + 0x90400200, 0x00080018, 0x00200800, 0x40801004, + 0x90440200, 0x00080018, 0x04200800, 0x40901004, + 0x90000600, 0x00080030, 0x08200000, 0x00801104, + 0x90040600, 0x00080030, 0x0c200000, 0x00901104, + 0x90400600, 0x00080038, 0x08200800, 0x40801104, + 0x90440600, 0x00080038, 0x0c200800, 0x40901104, + 0x00000002, 0x00002000, 0x20000000, 0x00000001, + 0x00040002, 0x00002000, 0x24000000, 0x00100001, + 0x00400002, 0x00002008, 0x20000800, 0x40000001, + 0x00440002, 0x00002008, 0x24000800, 0x40100001, + 0x00000402, 0x00002020, 0x28000000, 0x00000101, + 0x00040402, 0x00002020, 0x2c000000, 0x00100101, + 0x00400402, 0x00002028, 0x28000800, 0x40000101, + 0x00440402, 0x00002028, 0x2c000800, 0x40100101, + 0x80000002, 0x00002010, 0x20000000, 0x00800001, + 0x80040002, 0x00002010, 0x24000000, 0x00900001, + 0x80400002, 0x00002018, 0x20000800, 0x40800001, + 0x80440002, 0x00002018, 0x24000800, 0x40900001, + 0x80000402, 0x00002030, 0x28000000, 0x00800101, + 0x80040402, 0x00002030, 0x2c000000, 0x00900101, + 0x80400402, 0x00002038, 0x28000800, 0x40800101, + 0x80440402, 0x00002038, 0x2c000800, 0x40900101, + 0x10000002, 0x00002000, 0x20200000, 0x00001001, + 0x10040002, 0x00002000, 0x24200000, 0x00101001, + 0x10400002, 0x00002008, 0x20200800, 0x40001001, + 0x10440002, 0x00002008, 0x24200800, 0x40101001, + 0x10000402, 0x00002020, 0x28200000, 0x00001101, + 0x10040402, 0x00002020, 0x2c200000, 0x00101101, + 0x10400402, 0x00002028, 0x28200800, 0x40001101, + 0x10440402, 0x00002028, 0x2c200800, 0x40101101, + 0x90000002, 0x00002010, 0x20200000, 0x00801001, + 0x90040002, 0x00002010, 0x24200000, 0x00901001, + 0x90400002, 0x00002018, 0x20200800, 0x40801001, + 0x90440002, 0x00002018, 0x24200800, 0x40901001, + 0x90000402, 0x00002030, 0x28200000, 0x00801101, + 0x90040402, 0x00002030, 0x2c200000, 0x00901101, + 0x90400402, 0x00002038, 0x28200800, 0x40801101, + 0x90440402, 0x00002038, 0x2c200800, 0x40901101, + 0x00000202, 0x00082000, 0x20000000, 0x00000005, + 0x00040202, 0x00082000, 0x24000000, 0x00100005, + 0x00400202, 0x00082008, 0x20000800, 0x40000005, + 0x00440202, 0x00082008, 0x24000800, 0x40100005, + 0x00000602, 0x00082020, 0x28000000, 0x00000105, + 0x00040602, 0x00082020, 0x2c000000, 0x00100105, + 0x00400602, 0x00082028, 0x28000800, 0x40000105, + 0x00440602, 0x00082028, 0x2c000800, 0x40100105, + 0x80000202, 0x00082010, 0x20000000, 0x00800005, + 0x80040202, 0x00082010, 0x24000000, 0x00900005, + 0x80400202, 0x00082018, 0x20000800, 0x40800005, + 0x80440202, 0x00082018, 0x24000800, 0x40900005, + 0x80000602, 0x00082030, 0x28000000, 0x00800105, + 0x80040602, 0x00082030, 0x2c000000, 0x00900105, + 0x80400602, 0x00082038, 0x28000800, 0x40800105, + 0x80440602, 0x00082038, 0x2c000800, 0x40900105, + 0x10000202, 0x00082000, 0x20200000, 0x00001005, + 0x10040202, 0x00082000, 0x24200000, 0x00101005, + 0x10400202, 0x00082008, 0x20200800, 0x40001005, + 0x10440202, 0x00082008, 0x24200800, 0x40101005, + 0x10000602, 0x00082020, 0x28200000, 0x00001105, + 0x10040602, 0x00082020, 0x2c200000, 0x00101105, + 0x10400602, 0x00082028, 0x28200800, 0x40001105, + 0x10440602, 0x00082028, 0x2c200800, 0x40101105, + 0x90000202, 0x00082010, 0x20200000, 0x00801005, + 0x90040202, 0x00082010, 0x24200000, 0x00901005, + 0x90400202, 0x00082018, 0x20200800, 0x40801005, + 0x90440202, 0x00082018, 0x24200800, 0x40901005, + 0x90000602, 0x00082030, 0x28200000, 0x00801105, + 0x90040602, 0x00082030, 0x2c200000, 0x00901105, + 0x90400602, 0x00082038, 0x28200800, 0x40801105, + 0x90440602, 0x00082038, 0x2c200800, 0x40901105, + + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000008, 0x00080000, 0x10000000, + 0x02000000, 0x00000000, 0x00000080, 0x00001000, + 0x02000000, 0x00000008, 0x00080080, 0x10001000, + 0x00004000, 0x00000000, 0x00000040, 0x00040000, + 0x00004000, 0x00000008, 0x00080040, 0x10040000, + 0x02004000, 0x00000000, 0x000000c0, 0x00041000, + 0x02004000, 0x00000008, 0x000800c0, 0x10041000, + 0x00020000, 0x00008000, 0x08000000, 0x00200000, + 0x00020000, 0x00008008, 0x08080000, 0x10200000, + 0x02020000, 0x00008000, 0x08000080, 0x00201000, + 0x02020000, 0x00008008, 0x08080080, 0x10201000, + 0x00024000, 0x00008000, 0x08000040, 0x00240000, + 0x00024000, 0x00008008, 0x08080040, 0x10240000, + 0x02024000, 0x00008000, 0x080000c0, 0x00241000, + 0x02024000, 0x00008008, 0x080800c0, 0x10241000, + 0x00000000, 0x01000000, 0x00002000, 0x00000020, + 0x00000000, 0x01000008, 0x00082000, 0x10000020, + 0x02000000, 0x01000000, 0x00002080, 0x00001020, + 0x02000000, 0x01000008, 0x00082080, 0x10001020, + 0x00004000, 0x01000000, 0x00002040, 0x00040020, + 0x00004000, 0x01000008, 0x00082040, 0x10040020, + 0x02004000, 0x01000000, 0x000020c0, 0x00041020, + 0x02004000, 0x01000008, 0x000820c0, 0x10041020, + 0x00020000, 0x01008000, 0x08002000, 0x00200020, + 0x00020000, 0x01008008, 0x08082000, 0x10200020, + 0x02020000, 0x01008000, 0x08002080, 0x00201020, + 0x02020000, 0x01008008, 0x08082080, 0x10201020, + 0x00024000, 0x01008000, 0x08002040, 0x00240020, + 0x00024000, 0x01008008, 0x08082040, 0x10240020, + 0x02024000, 0x01008000, 0x080020c0, 0x00241020, + 0x02024000, 0x01008008, 0x080820c0, 0x10241020, + 0x00000400, 0x04000000, 0x00100000, 0x00000004, + 0x00000400, 0x04000008, 0x00180000, 0x10000004, + 0x02000400, 0x04000000, 0x00100080, 0x00001004, + 0x02000400, 0x04000008, 0x00180080, 0x10001004, + 0x00004400, 0x04000000, 0x00100040, 0x00040004, + 0x00004400, 0x04000008, 0x00180040, 0x10040004, + 0x02004400, 0x04000000, 0x001000c0, 0x00041004, + 0x02004400, 0x04000008, 0x001800c0, 0x10041004, + 0x00020400, 0x04008000, 0x08100000, 0x00200004, + 0x00020400, 0x04008008, 0x08180000, 0x10200004, + 0x02020400, 0x04008000, 0x08100080, 0x00201004, + 0x02020400, 0x04008008, 0x08180080, 0x10201004, + 0x00024400, 0x04008000, 0x08100040, 0x00240004, + 0x00024400, 0x04008008, 0x08180040, 0x10240004, + 0x02024400, 0x04008000, 0x081000c0, 0x00241004, + 0x02024400, 0x04008008, 0x081800c0, 0x10241004, + 0x00000400, 0x05000000, 0x00102000, 0x00000024, + 0x00000400, 0x05000008, 0x00182000, 0x10000024, + 0x02000400, 0x05000000, 0x00102080, 0x00001024, + 0x02000400, 0x05000008, 0x00182080, 0x10001024, + 0x00004400, 0x05000000, 0x00102040, 0x00040024, + 0x00004400, 0x05000008, 0x00182040, 0x10040024, + 0x02004400, 0x05000000, 0x001020c0, 0x00041024, + 0x02004400, 0x05000008, 0x001820c0, 0x10041024, + 0x00020400, 0x05008000, 0x08102000, 0x00200024, + 0x00020400, 0x05008008, 0x08182000, 0x10200024, + 0x02020400, 0x05008000, 0x08102080, 0x00201024, + 0x02020400, 0x05008008, 0x08182080, 0x10201024, + 0x00024400, 0x05008000, 0x08102040, 0x00240024, + 0x00024400, 0x05008008, 0x08182040, 0x10240024, + 0x02024400, 0x05008000, 0x081020c0, 0x00241024, + 0x02024400, 0x05008008, 0x081820c0, 0x10241024, + 0x00000800, 0x00010000, 0x20000000, 0x00000010, + 0x00000800, 0x00010008, 0x20080000, 0x10000010, + 0x02000800, 0x00010000, 0x20000080, 0x00001010, + 0x02000800, 0x00010008, 0x20080080, 0x10001010, + 0x00004800, 0x00010000, 0x20000040, 0x00040010, + 0x00004800, 0x00010008, 0x20080040, 0x10040010, + 0x02004800, 0x00010000, 0x200000c0, 0x00041010, + 0x02004800, 0x00010008, 0x200800c0, 0x10041010, + 0x00020800, 0x00018000, 0x28000000, 0x00200010, + 0x00020800, 0x00018008, 0x28080000, 0x10200010, + 0x02020800, 0x00018000, 0x28000080, 0x00201010, + 0x02020800, 0x00018008, 0x28080080, 0x10201010, + 0x00024800, 0x00018000, 0x28000040, 0x00240010, + 0x00024800, 0x00018008, 0x28080040, 0x10240010, + 0x02024800, 0x00018000, 0x280000c0, 0x00241010, + 0x02024800, 0x00018008, 0x280800c0, 0x10241010, + 0x00000800, 0x01010000, 0x20002000, 0x00000030, + 0x00000800, 0x01010008, 0x20082000, 0x10000030, + 0x02000800, 0x01010000, 0x20002080, 0x00001030, + 0x02000800, 0x01010008, 0x20082080, 0x10001030, + 0x00004800, 0x01010000, 0x20002040, 0x00040030, + 0x00004800, 0x01010008, 0x20082040, 0x10040030, + 0x02004800, 0x01010000, 0x200020c0, 0x00041030, + 0x02004800, 0x01010008, 0x200820c0, 0x10041030, + 0x00020800, 0x01018000, 0x28002000, 0x00200030, + 0x00020800, 0x01018008, 0x28082000, 0x10200030, + 0x02020800, 0x01018000, 0x28002080, 0x00201030, + 0x02020800, 0x01018008, 0x28082080, 0x10201030, + 0x00024800, 0x01018000, 0x28002040, 0x00240030, + 0x00024800, 0x01018008, 0x28082040, 0x10240030, + 0x02024800, 0x01018000, 0x280020c0, 0x00241030, + 0x02024800, 0x01018008, 0x280820c0, 0x10241030, + 0x00000c00, 0x04010000, 0x20100000, 0x00000014, + 0x00000c00, 0x04010008, 0x20180000, 0x10000014, + 0x02000c00, 0x04010000, 0x20100080, 0x00001014, + 0x02000c00, 0x04010008, 0x20180080, 0x10001014, + 0x00004c00, 0x04010000, 0x20100040, 0x00040014, + 0x00004c00, 0x04010008, 0x20180040, 0x10040014, + 0x02004c00, 0x04010000, 0x201000c0, 0x00041014, + 0x02004c00, 0x04010008, 0x201800c0, 0x10041014, + 0x00020c00, 0x04018000, 0x28100000, 0x00200014, + 0x00020c00, 0x04018008, 0x28180000, 0x10200014, + 0x02020c00, 0x04018000, 0x28100080, 0x00201014, + 0x02020c00, 0x04018008, 0x28180080, 0x10201014, + 0x00024c00, 0x04018000, 0x28100040, 0x00240014, + 0x00024c00, 0x04018008, 0x28180040, 0x10240014, + 0x02024c00, 0x04018000, 0x281000c0, 0x00241014, + 0x02024c00, 0x04018008, 0x281800c0, 0x10241014, + 0x00000c00, 0x05010000, 0x20102000, 0x00000034, + 0x00000c00, 0x05010008, 0x20182000, 0x10000034, + 0x02000c00, 0x05010000, 0x20102080, 0x00001034, + 0x02000c00, 0x05010008, 0x20182080, 0x10001034, + 0x00004c00, 0x05010000, 0x20102040, 0x00040034, + 0x00004c00, 0x05010008, 0x20182040, 0x10040034, + 0x02004c00, 0x05010000, 0x201020c0, 0x00041034, + 0x02004c00, 0x05010008, 0x201820c0, 0x10041034, + 0x00020c00, 0x05018000, 0x28102000, 0x00200034, + 0x00020c00, 0x05018008, 0x28182000, 0x10200034, + 0x02020c00, 0x05018000, 0x28102080, 0x00201034, + 0x02020c00, 0x05018008, 0x28182080, 0x10201034, + 0x00024c00, 0x05018000, 0x28102040, 0x00240034, + 0x00024c00, 0x05018008, 0x28182040, 0x10240034, + 0x02024c00, 0x05018000, 0x281020c0, 0x00241034, + 0x02024c00, 0x05018008, 0x281820c0, 0x10241034 +}; + +/* S-box lookup tables */ + +static const u32 S1[64] = { + 0x01010400, 0x00000000, 0x00010000, 0x01010404, + 0x01010004, 0x00010404, 0x00000004, 0x00010000, + 0x00000400, 0x01010400, 0x01010404, 0x00000400, + 0x01000404, 0x01010004, 0x01000000, 0x00000004, + 0x00000404, 0x01000400, 0x01000400, 0x00010400, + 0x00010400, 0x01010000, 0x01010000, 0x01000404, + 0x00010004, 0x01000004, 0x01000004, 0x00010004, + 0x00000000, 0x00000404, 0x00010404, 0x01000000, + 0x00010000, 0x01010404, 0x00000004, 0x01010000, + 0x01010400, 0x01000000, 0x01000000, 0x00000400, + 0x01010004, 0x00010000, 0x00010400, 0x01000004, + 0x00000400, 0x00000004, 0x01000404, 0x00010404, + 0x01010404, 0x00010004, 0x01010000, 0x01000404, + 0x01000004, 0x00000404, 0x00010404, 0x01010400, + 0x00000404, 0x01000400, 0x01000400, 0x00000000, + 0x00010004, 0x00010400, 0x00000000, 0x01010004 +}; + +static const u32 S2[64] = { + 0x80108020, 0x80008000, 0x00008000, 0x00108020, + 0x00100000, 0x00000020, 0x80100020, 0x80008020, + 0x80000020, 0x80108020, 0x80108000, 0x80000000, + 0x80008000, 0x00100000, 0x00000020, 0x80100020, + 0x00108000, 0x00100020, 0x80008020, 0x00000000, + 0x80000000, 0x00008000, 0x00108020, 0x80100000, + 0x00100020, 0x80000020, 0x00000000, 0x00108000, + 0x00008020, 0x80108000, 0x80100000, 0x00008020, + 0x00000000, 0x00108020, 0x80100020, 0x00100000, + 0x80008020, 0x80100000, 0x80108000, 0x00008000, + 0x80100000, 0x80008000, 0x00000020, 0x80108020, + 0x00108020, 0x00000020, 0x00008000, 0x80000000, + 0x00008020, 0x80108000, 0x00100000, 0x80000020, + 0x00100020, 0x80008020, 0x80000020, 0x00100020, + 0x00108000, 0x00000000, 0x80008000, 0x00008020, + 0x80000000, 0x80100020, 0x80108020, 0x00108000 +}; + +static const u32 S3[64] = { + 0x00000208, 0x08020200, 0x00000000, 0x08020008, + 0x08000200, 0x00000000, 0x00020208, 0x08000200, + 0x00020008, 0x08000008, 0x08000008, 0x00020000, + 0x08020208, 0x00020008, 0x08020000, 0x00000208, + 0x08000000, 0x00000008, 0x08020200, 0x00000200, + 0x00020200, 0x08020000, 0x08020008, 0x00020208, + 0x08000208, 0x00020200, 0x00020000, 0x08000208, + 0x00000008, 0x08020208, 0x00000200, 0x08000000, + 0x08020200, 0x08000000, 0x00020008, 0x00000208, + 0x00020000, 0x08020200, 0x08000200, 0x00000000, + 0x00000200, 0x00020008, 0x08020208, 0x08000200, + 0x08000008, 0x00000200, 0x00000000, 0x08020008, + 0x08000208, 0x00020000, 0x08000000, 0x08020208, + 0x00000008, 0x00020208, 0x00020200, 0x08000008, + 0x08020000, 0x08000208, 0x00000208, 0x08020000, + 0x00020208, 0x00000008, 0x08020008, 0x00020200 +}; + +static const u32 S4[64] = { + 0x00802001, 0x00002081, 0x00002081, 0x00000080, + 0x00802080, 0x00800081, 0x00800001, 0x00002001, + 0x00000000, 0x00802000, 0x00802000, 0x00802081, + 0x00000081, 0x00000000, 0x00800080, 0x00800001, + 0x00000001, 0x00002000, 0x00800000, 0x00802001, + 0x00000080, 0x00800000, 0x00002001, 0x00002080, + 0x00800081, 0x00000001, 0x00002080, 0x00800080, + 0x00002000, 0x00802080, 0x00802081, 0x00000081, + 0x00800080, 0x00800001, 0x00802000, 0x00802081, + 0x00000081, 0x00000000, 0x00000000, 0x00802000, + 0x00002080, 0x00800080, 0x00800081, 0x00000001, + 0x00802001, 0x00002081, 0x00002081, 0x00000080, + 0x00802081, 0x00000081, 0x00000001, 0x00002000, + 0x00800001, 0x00002001, 0x00802080, 0x00800081, + 0x00002001, 0x00002080, 0x00800000, 0x00802001, + 0x00000080, 0x00800000, 0x00002000, 0x00802080 +}; + +static const u32 S5[64] = { + 0x00000100, 0x02080100, 0x02080000, 0x42000100, + 0x00080000, 0x00000100, 0x40000000, 0x02080000, + 0x40080100, 0x00080000, 0x02000100, 0x40080100, + 0x42000100, 0x42080000, 0x00080100, 0x40000000, + 0x02000000, 0x40080000, 0x40080000, 0x00000000, + 0x40000100, 0x42080100, 0x42080100, 0x02000100, + 0x42080000, 0x40000100, 0x00000000, 0x42000000, + 0x02080100, 0x02000000, 0x42000000, 0x00080100, + 0x00080000, 0x42000100, 0x00000100, 0x02000000, + 0x40000000, 0x02080000, 0x42000100, 0x40080100, + 0x02000100, 0x40000000, 0x42080000, 0x02080100, + 0x40080100, 0x00000100, 0x02000000, 0x42080000, + 0x42080100, 0x00080100, 0x42000000, 0x42080100, + 0x02080000, 0x00000000, 0x40080000, 0x42000000, + 0x00080100, 0x02000100, 0x40000100, 0x00080000, + 0x00000000, 0x40080000, 0x02080100, 0x40000100 +}; + +static const u32 S6[64] = { + 0x20000010, 0x20400000, 0x00004000, 0x20404010, + 0x20400000, 0x00000010, 0x20404010, 0x00400000, + 0x20004000, 0x00404010, 0x00400000, 0x20000010, + 0x00400010, 0x20004000, 0x20000000, 0x00004010, + 0x00000000, 0x00400010, 0x20004010, 0x00004000, + 0x00404000, 0x20004010, 0x00000010, 0x20400010, + 0x20400010, 0x00000000, 0x00404010, 0x20404000, + 0x00004010, 0x00404000, 0x20404000, 0x20000000, + 0x20004000, 0x00000010, 0x20400010, 0x00404000, + 0x20404010, 0x00400000, 0x00004010, 0x20000010, + 0x00400000, 0x20004000, 0x20000000, 0x00004010, + 0x20000010, 0x20404010, 0x00404000, 0x20400000, + 0x00404010, 0x20404000, 0x00000000, 0x20400010, + 0x00000010, 0x00004000, 0x20400000, 0x00404010, + 0x00004000, 0x00400010, 0x20004010, 0x00000000, + 0x20404000, 0x20000000, 0x00400010, 0x20004010 +}; + +static const u32 S7[64] = { + 0x00200000, 0x04200002, 0x04000802, 0x00000000, + 0x00000800, 0x04000802, 0x00200802, 0x04200800, + 0x04200802, 0x00200000, 0x00000000, 0x04000002, + 0x00000002, 0x04000000, 0x04200002, 0x00000802, + 0x04000800, 0x00200802, 0x00200002, 0x04000800, + 0x04000002, 0x04200000, 0x04200800, 0x00200002, + 0x04200000, 0x00000800, 0x00000802, 0x04200802, + 0x00200800, 0x00000002, 0x04000000, 0x00200800, + 0x04000000, 0x00200800, 0x00200000, 0x04000802, + 0x04000802, 0x04200002, 0x04200002, 0x00000002, + 0x00200002, 0x04000000, 0x04000800, 0x00200000, + 0x04200800, 0x00000802, 0x00200802, 0x04200800, + 0x00000802, 0x04000002, 0x04200802, 0x04200000, + 0x00200800, 0x00000000, 0x00000002, 0x04200802, + 0x00000000, 0x00200802, 0x04200000, 0x00000800, + 0x04000002, 0x04000800, 0x00000800, 0x00200002 +}; + +static const u32 S8[64] = { + 0x10001040, 0x00001000, 0x00040000, 0x10041040, + 0x10000000, 0x10001040, 0x00000040, 0x10000000, + 0x00040040, 0x10040000, 0x10041040, 0x00041000, + 0x10041000, 0x00041040, 0x00001000, 0x00000040, + 0x10040000, 0x10000040, 0x10001000, 0x00001040, + 0x00041000, 0x00040040, 0x10040040, 0x10041000, + 0x00001040, 0x00000000, 0x00000000, 0x10040040, + 0x10000040, 0x10001000, 0x00041040, 0x00040000, + 0x00041040, 0x00040000, 0x10041000, 0x00001000, + 0x00000040, 0x10040040, 0x00001000, 0x00041040, + 0x10001000, 0x00000040, 0x10000040, 0x10040000, + 0x10040040, 0x10000000, 0x00040000, 0x10001040, + 0x00000000, 0x10041040, 0x00040040, 0x10000040, + 0x10040000, 0x10001000, 0x10001040, 0x00000000, + 0x10041040, 0x00041000, 0x00041000, 0x00001040, + 0x00001040, 0x00040040, 0x10000000, 0x10041000 +}; + +/* Encryption components: IP, FP, and round function */ + +#define IP(L, R, T) \ + ROL(R, 4); \ + T = L; \ + L ^= R; \ + L &= 0xf0f0f0f0; \ + R ^= L; \ + L ^= T; \ + ROL(R, 12); \ + T = L; \ + L ^= R; \ + L &= 0xffff0000; \ + R ^= L; \ + L ^= T; \ + ROR(R, 14); \ + T = L; \ + L ^= R; \ + L &= 0xcccccccc; \ + R ^= L; \ + L ^= T; \ + ROL(R, 6); \ + T = L; \ + L ^= R; \ + L &= 0xff00ff00; \ + R ^= L; \ + L ^= T; \ + ROR(R, 7); \ + T = L; \ + L ^= R; \ + L &= 0xaaaaaaaa; \ + R ^= L; \ + L ^= T; \ + ROL(L, 1); + +#define FP(L, R, T) \ + ROR(L, 1); \ + T = L; \ + L ^= R; \ + L &= 0xaaaaaaaa; \ + R ^= L; \ + L ^= T; \ + ROL(R, 7); \ + T = L; \ + L ^= R; \ + L &= 0xff00ff00; \ + R ^= L; \ + L ^= T; \ + ROR(R, 6); \ + T = L; \ + L ^= R; \ + L &= 0xcccccccc; \ + R ^= L; \ + L ^= T; \ + ROL(R, 14); \ + T = L; \ + L ^= R; \ + L &= 0xffff0000; \ + R ^= L; \ + L ^= T; \ + ROR(R, 12); \ + T = L; \ + L ^= R; \ + L &= 0xf0f0f0f0; \ + R ^= L; \ + L ^= T; \ + ROR(R, 4); + +#define ROUND(L, R, A, B, K, d) \ + B = K[0]; A = K[1]; K += d; \ + B ^= R; A ^= R; \ + B &= 0x3f3f3f3f; ROR(A, 4); \ + L ^= S8[0xff & B]; A &= 0x3f3f3f3f; \ + L ^= S6[0xff & (B >> 8)]; B >>= 16; \ + L ^= S7[0xff & A]; \ + L ^= S5[0xff & (A >> 8)]; A >>= 16; \ + L ^= S4[0xff & B]; \ + L ^= S2[0xff & (B >> 8)]; \ + L ^= S3[0xff & A]; \ + L ^= S1[0xff & (A >> 8)]; + +/* + * PC2 lookup tables are organized as 2 consecutive sets of 4 interleaved + * tables of 128 elements. One set is for C_i and the other for D_i, while + * the 4 interleaved tables correspond to four 7-bit subsets of C_i or D_i. + * + * After PC1 each of the variables a,b,c,d contains a 7 bit subset of C_i + * or D_i in bits 7-1 (bit 0 being the least significant). + */ + +#define T1(x) pt[2 * (x) + 0] +#define T2(x) pt[2 * (x) + 1] +#define T3(x) pt[2 * (x) + 2] +#define T4(x) pt[2 * (x) + 3] + +#define DES_PC2(a, b, c, d) (T4(d) | T3(c) | T2(b) | T1(a)) + +/* + * Encryption key expansion + * + * RFC2451: Weak key checks SHOULD be performed. + * + * FIPS 74: + * + * Keys having duals are keys which produce all zeros, all ones, or + * alternating zero-one patterns in the C and D registers after Permuted + * Choice 1 has operated on the key. + * + */ +static unsigned long des_ekey(u32 *pe, const u8 *k) +{ + /* K&R: long is at least 32 bits */ + unsigned long a, b, c, d, w; + const u32 *pt = pc2; + + d = k[4]; d &= 0x0e; d <<= 4; d |= k[0] & 0x1e; d = pc1[d]; + c = k[5]; c &= 0x0e; c <<= 4; c |= k[1] & 0x1e; c = pc1[c]; + b = k[6]; b &= 0x0e; b <<= 4; b |= k[2] & 0x1e; b = pc1[b]; + a = k[7]; a &= 0x0e; a <<= 4; a |= k[3] & 0x1e; a = pc1[a]; + + pe[15 * 2 + 0] = DES_PC2(a, b, c, d); d = rs[d]; + pe[14 * 2 + 0] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[13 * 2 + 0] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[12 * 2 + 0] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[11 * 2 + 0] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[10 * 2 + 0] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 9 * 2 + 0] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 8 * 2 + 0] = DES_PC2(d, a, b, c); c = rs[c]; + pe[ 7 * 2 + 0] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 6 * 2 + 0] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 5 * 2 + 0] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 4 * 2 + 0] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 3 * 2 + 0] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 2 * 2 + 0] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 1 * 2 + 0] = DES_PC2(c, d, a, b); b = rs[b]; + pe[ 0 * 2 + 0] = DES_PC2(b, c, d, a); + + /* Check if first half is weak */ + w = (a ^ c) | (b ^ d) | (rs[a] ^ c) | (b ^ rs[d]); + + /* Skip to next table set */ + pt += 512; + + d = k[0]; d &= 0xe0; d >>= 4; d |= k[4] & 0xf0; d = pc1[d + 1]; + c = k[1]; c &= 0xe0; c >>= 4; c |= k[5] & 0xf0; c = pc1[c + 1]; + b = k[2]; b &= 0xe0; b >>= 4; b |= k[6] & 0xf0; b = pc1[b + 1]; + a = k[3]; a &= 0xe0; a >>= 4; a |= k[7] & 0xf0; a = pc1[a + 1]; + + /* Check if second half is weak */ + w |= (a ^ c) | (b ^ d) | (rs[a] ^ c) | (b ^ rs[d]); + + pe[15 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; + pe[14 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[13 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[12 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[11 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[10 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 9 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 8 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; + pe[ 7 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 6 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 5 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 4 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 3 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 2 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 1 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; + pe[ 0 * 2 + 1] = DES_PC2(b, c, d, a); + + /* Fixup: 2413 5768 -> 1357 2468 */ + for (d = 0; d < 16; ++d) { + a = pe[2 * d]; + b = pe[2 * d + 1]; + c = a ^ b; + c &= 0xffff0000; + a ^= c; + b ^= c; + ROL(b, 18); + pe[2 * d] = a; + pe[2 * d + 1] = b; + } + + /* Zero if weak key */ + return w; +} + +int des_expand_key(struct des_ctx *ctx, const u8 *key, unsigned int keylen) +{ + if (keylen != DES_KEY_SIZE) + return -EINVAL; + + return des_ekey(ctx->expkey, key) ? 0 : -ENOKEY; +} +EXPORT_SYMBOL_GPL(des_expand_key); + +/* + * Decryption key expansion + * + * No weak key checking is performed, as this is only used by triple DES + * + */ +static void dkey(u32 *pe, const u8 *k) +{ + /* K&R: long is at least 32 bits */ + unsigned long a, b, c, d; + const u32 *pt = pc2; + + d = k[4]; d &= 0x0e; d <<= 4; d |= k[0] & 0x1e; d = pc1[d]; + c = k[5]; c &= 0x0e; c <<= 4; c |= k[1] & 0x1e; c = pc1[c]; + b = k[6]; b &= 0x0e; b <<= 4; b |= k[2] & 0x1e; b = pc1[b]; + a = k[7]; a &= 0x0e; a <<= 4; a |= k[3] & 0x1e; a = pc1[a]; + + pe[ 0 * 2] = DES_PC2(a, b, c, d); d = rs[d]; + pe[ 1 * 2] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 2 * 2] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 3 * 2] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 4 * 2] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 5 * 2] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 6 * 2] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 7 * 2] = DES_PC2(d, a, b, c); c = rs[c]; + pe[ 8 * 2] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 9 * 2] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[10 * 2] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[11 * 2] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[12 * 2] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[13 * 2] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[14 * 2] = DES_PC2(c, d, a, b); b = rs[b]; + pe[15 * 2] = DES_PC2(b, c, d, a); + + /* Skip to next table set */ + pt += 512; + + d = k[0]; d &= 0xe0; d >>= 4; d |= k[4] & 0xf0; d = pc1[d + 1]; + c = k[1]; c &= 0xe0; c >>= 4; c |= k[5] & 0xf0; c = pc1[c + 1]; + b = k[2]; b &= 0xe0; b >>= 4; b |= k[6] & 0xf0; b = pc1[b + 1]; + a = k[3]; a &= 0xe0; a >>= 4; a |= k[7] & 0xf0; a = pc1[a + 1]; + + pe[ 0 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; + pe[ 1 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 2 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 3 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 4 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 5 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 6 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 7 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; + pe[ 8 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 9 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[10 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[11 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[12 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[13 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[14 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; + pe[15 * 2 + 1] = DES_PC2(b, c, d, a); + + /* Fixup: 2413 5768 -> 1357 2468 */ + for (d = 0; d < 16; ++d) { + a = pe[2 * d]; + b = pe[2 * d + 1]; + c = a ^ b; + c &= 0xffff0000; + a ^= c; + b ^= c; + ROL(b, 18); + pe[2 * d] = a; + pe[2 * d + 1] = b; + } +} + +void des_encrypt(const struct des_ctx *ctx, u8 *dst, const u8 *src) +{ + const u32 *K = ctx->expkey; + u32 L, R, A, B; + int i; + + L = get_unaligned_le32(src); + R = get_unaligned_le32(src + 4); + + IP(L, R, A); + for (i = 0; i < 8; i++) { + ROUND(L, R, A, B, K, 2); + ROUND(R, L, A, B, K, 2); + } + FP(R, L, A); + + put_unaligned_le32(R, dst); + put_unaligned_le32(L, dst + 4); +} +EXPORT_SYMBOL_GPL(des_encrypt); + +void des_decrypt(const struct des_ctx *ctx, u8 *dst, const u8 *src) +{ + const u32 *K = ctx->expkey + DES_EXPKEY_WORDS - 2; + u32 L, R, A, B; + int i; + + L = get_unaligned_le32(src); + R = get_unaligned_le32(src + 4); + + IP(L, R, A); + for (i = 0; i < 8; i++) { + ROUND(L, R, A, B, K, -2); + ROUND(R, L, A, B, K, -2); + } + FP(R, L, A); + + put_unaligned_le32(R, dst); + put_unaligned_le32(L, dst + 4); +} +EXPORT_SYMBOL_GPL(des_decrypt); + +int des3_ede_expand_key(struct des3_ede_ctx *ctx, const u8 *key, + unsigned int keylen) +{ + u32 *pe = ctx->expkey; + int err; + + if (keylen != DES3_EDE_KEY_SIZE) + return -EINVAL; + + err = des3_ede_verify_key(key, keylen, true); + if (err && err != -ENOKEY) + return err; + + des_ekey(pe, key); pe += DES_EXPKEY_WORDS; key += DES_KEY_SIZE; + dkey(pe, key); pe += DES_EXPKEY_WORDS; key += DES_KEY_SIZE; + des_ekey(pe, key); + + return err; +} +EXPORT_SYMBOL_GPL(des3_ede_expand_key); + +void des3_ede_encrypt(const struct des3_ede_ctx *dctx, u8 *dst, const u8 *src) +{ + const u32 *K = dctx->expkey; + u32 L, R, A, B; + int i; + + L = get_unaligned_le32(src); + R = get_unaligned_le32(src + 4); + + IP(L, R, A); + for (i = 0; i < 8; i++) { + ROUND(L, R, A, B, K, 2); + ROUND(R, L, A, B, K, 2); + } + for (i = 0; i < 8; i++) { + ROUND(R, L, A, B, K, 2); + ROUND(L, R, A, B, K, 2); + } + for (i = 0; i < 8; i++) { + ROUND(L, R, A, B, K, 2); + ROUND(R, L, A, B, K, 2); + } + FP(R, L, A); + + put_unaligned_le32(R, dst); + put_unaligned_le32(L, dst + 4); +} +EXPORT_SYMBOL_GPL(des3_ede_encrypt); + +void des3_ede_decrypt(const struct des3_ede_ctx *dctx, u8 *dst, const u8 *src) +{ + const u32 *K = dctx->expkey + DES3_EDE_EXPKEY_WORDS - 2; + u32 L, R, A, B; + int i; + + L = get_unaligned_le32(src); + R = get_unaligned_le32(src + 4); + + IP(L, R, A); + for (i = 0; i < 8; i++) { + ROUND(L, R, A, B, K, -2); + ROUND(R, L, A, B, K, -2); + } + for (i = 0; i < 8; i++) { + ROUND(R, L, A, B, K, -2); + ROUND(L, R, A, B, K, -2); + } + for (i = 0; i < 8; i++) { + ROUND(L, R, A, B, K, -2); + ROUND(R, L, A, B, K, -2); + } + FP(R, L, A); + + put_unaligned_le32(R, dst); + put_unaligned_le32(L, dst + 4); +} +EXPORT_SYMBOL_GPL(des3_ede_decrypt); + +MODULE_LICENSE("GPL"); diff --git a/lib/sha256.c b/lib/crypto/sha256.c index d9af148d4349..66cb04b0cf4e 100644 --- a/lib/sha256.c +++ b/lib/crypto/sha256.c @@ -12,9 +12,11 @@ */ #include <linux/bitops.h> -#include <linux/sha256.h> +#include <linux/export.h> +#include <linux/module.h> #include <linux/string.h> -#include <asm/byteorder.h> +#include <crypto/sha.h> +#include <asm/unaligned.h> static inline u32 Ch(u32 x, u32 y, u32 z) { @@ -33,7 +35,7 @@ static inline u32 Maj(u32 x, u32 y, u32 z) static inline void LOAD_OP(int I, u32 *W, const u8 *input) { - W[I] = __be32_to_cpu(((__be32 *)(input))[I]); + W[I] = get_unaligned_be32((__u32 *)input + I); } static inline void BLEND_OP(int I, u32 *W) @@ -92,131 +94,116 @@ static void sha256_transform(u32 *state, const u8 *input) t1 = b + e1(g) + Ch(g, h, a) + 0x9bdc06a7 + W[14]; t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; t1 = a + e1(f) + Ch(f, g, h) + 0xc19bf174 + W[15]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1+t2; + t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; t1 = h + e1(e) + Ch(e, f, g) + 0xe49b69c1 + W[16]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1+t2; + t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; t1 = g + e1(d) + Ch(d, e, f) + 0xefbe4786 + W[17]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1+t2; + t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; t1 = f + e1(c) + Ch(c, d, e) + 0x0fc19dc6 + W[18]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1+t2; + t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; t1 = e + e1(b) + Ch(b, c, d) + 0x240ca1cc + W[19]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1+t2; + t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; t1 = d + e1(a) + Ch(a, b, c) + 0x2de92c6f + W[20]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1+t2; + t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; t1 = c + e1(h) + Ch(h, a, b) + 0x4a7484aa + W[21]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1+t2; + t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; t1 = b + e1(g) + Ch(g, h, a) + 0x5cb0a9dc + W[22]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1+t2; + t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; t1 = a + e1(f) + Ch(f, g, h) + 0x76f988da + W[23]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1+t2; + t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; t1 = h + e1(e) + Ch(e, f, g) + 0x983e5152 + W[24]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1+t2; + t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; t1 = g + e1(d) + Ch(d, e, f) + 0xa831c66d + W[25]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1+t2; + t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; t1 = f + e1(c) + Ch(c, d, e) + 0xb00327c8 + W[26]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1+t2; + t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; t1 = e + e1(b) + Ch(b, c, d) + 0xbf597fc7 + W[27]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1+t2; + t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; t1 = d + e1(a) + Ch(a, b, c) + 0xc6e00bf3 + W[28]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1+t2; + t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; t1 = c + e1(h) + Ch(h, a, b) + 0xd5a79147 + W[29]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1+t2; + t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; t1 = b + e1(g) + Ch(g, h, a) + 0x06ca6351 + W[30]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1+t2; + t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; t1 = a + e1(f) + Ch(f, g, h) + 0x14292967 + W[31]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1+t2; + t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; t1 = h + e1(e) + Ch(e, f, g) + 0x27b70a85 + W[32]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1+t2; + t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; t1 = g + e1(d) + Ch(d, e, f) + 0x2e1b2138 + W[33]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1+t2; + t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; t1 = f + e1(c) + Ch(c, d, e) + 0x4d2c6dfc + W[34]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1+t2; + t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; t1 = e + e1(b) + Ch(b, c, d) + 0x53380d13 + W[35]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1+t2; + t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; t1 = d + e1(a) + Ch(a, b, c) + 0x650a7354 + W[36]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1+t2; + t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; t1 = c + e1(h) + Ch(h, a, b) + 0x766a0abb + W[37]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1+t2; + t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; t1 = b + e1(g) + Ch(g, h, a) + 0x81c2c92e + W[38]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1+t2; + t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; t1 = a + e1(f) + Ch(f, g, h) + 0x92722c85 + W[39]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1+t2; + t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; t1 = h + e1(e) + Ch(e, f, g) + 0xa2bfe8a1 + W[40]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1+t2; + t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; t1 = g + e1(d) + Ch(d, e, f) + 0xa81a664b + W[41]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1+t2; + t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; t1 = f + e1(c) + Ch(c, d, e) + 0xc24b8b70 + W[42]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1+t2; + t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; t1 = e + e1(b) + Ch(b, c, d) + 0xc76c51a3 + W[43]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1+t2; + t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; t1 = d + e1(a) + Ch(a, b, c) + 0xd192e819 + W[44]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1+t2; + t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; t1 = c + e1(h) + Ch(h, a, b) + 0xd6990624 + W[45]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1+t2; + t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; t1 = b + e1(g) + Ch(g, h, a) + 0xf40e3585 + W[46]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1+t2; + t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; t1 = a + e1(f) + Ch(f, g, h) + 0x106aa070 + W[47]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1+t2; + t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; t1 = h + e1(e) + Ch(e, f, g) + 0x19a4c116 + W[48]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1+t2; + t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; t1 = g + e1(d) + Ch(d, e, f) + 0x1e376c08 + W[49]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1+t2; + t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; t1 = f + e1(c) + Ch(c, d, e) + 0x2748774c + W[50]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1+t2; + t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; t1 = e + e1(b) + Ch(b, c, d) + 0x34b0bcb5 + W[51]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1+t2; + t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; t1 = d + e1(a) + Ch(a, b, c) + 0x391c0cb3 + W[52]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1+t2; + t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; t1 = c + e1(h) + Ch(h, a, b) + 0x4ed8aa4a + W[53]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1+t2; + t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; t1 = b + e1(g) + Ch(g, h, a) + 0x5b9cca4f + W[54]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1+t2; + t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; t1 = a + e1(f) + Ch(f, g, h) + 0x682e6ff3 + W[55]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1+t2; + t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; t1 = h + e1(e) + Ch(e, f, g) + 0x748f82ee + W[56]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1+t2; + t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; t1 = g + e1(d) + Ch(d, e, f) + 0x78a5636f + W[57]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1+t2; + t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; t1 = f + e1(c) + Ch(c, d, e) + 0x84c87814 + W[58]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1+t2; + t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; t1 = e + e1(b) + Ch(b, c, d) + 0x8cc70208 + W[59]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1+t2; + t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; t1 = d + e1(a) + Ch(a, b, c) + 0x90befffa + W[60]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1+t2; + t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; t1 = c + e1(h) + Ch(h, a, b) + 0xa4506ceb + W[61]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1+t2; + t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; t1 = b + e1(g) + Ch(g, h, a) + 0xbef9a3f7 + W[62]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1+t2; + t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; t1 = a + e1(f) + Ch(f, g, h) + 0xc67178f2 + W[63]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1+t2; + t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; state[0] += a; state[1] += b; state[2] += c; state[3] += d; state[4] += e; state[5] += f; state[6] += g; state[7] += h; /* clear any sensitive info... */ a = b = c = d = e = f = g = h = t1 = t2 = 0; - memset(W, 0, 64 * sizeof(u32)); -} - -int sha256_init(struct sha256_state *sctx) -{ - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - - return 0; + memzero_explicit(W, 64 * sizeof(u32)); } int sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) @@ -248,8 +235,15 @@ int sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) return 0; } +EXPORT_SYMBOL(sha256_update); -int sha256_final(struct sha256_state *sctx, u8 *out) +int sha224_update(struct sha256_state *sctx, const u8 *data, unsigned int len) +{ + return sha256_update(sctx, data, len); +} +EXPORT_SYMBOL(sha224_update); + +static int __sha256_final(struct sha256_state *sctx, u8 *out, int digest_words) { __be32 *dst = (__be32 *)out; __be64 bits; @@ -269,11 +263,25 @@ int sha256_final(struct sha256_state *sctx, u8 *out) sha256_update(sctx, (const u8 *)&bits, sizeof(bits)); /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be32(sctx->state[i]); + for (i = 0; i < digest_words; i++) + put_unaligned_be32(sctx->state[i], &dst[i]); /* Zeroize sensitive information. */ memset(sctx, 0, sizeof(*sctx)); return 0; } + +int sha256_final(struct sha256_state *sctx, u8 *out) +{ + return __sha256_final(sctx, out, 8); +} +EXPORT_SYMBOL(sha256_final); + +int sha224_final(struct sha256_state *sctx, u8 *out) +{ + return __sha256_final(sctx, out, 7); +} +EXPORT_SYMBOL(sha224_final); + +MODULE_LICENSE("GPL"); diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 55437fd5128b..61261195f5b6 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -25,16 +25,37 @@ #define ODEBUG_POOL_SIZE 1024 #define ODEBUG_POOL_MIN_LEVEL 256 +#define ODEBUG_POOL_PERCPU_SIZE 64 +#define ODEBUG_BATCH_SIZE 16 #define ODEBUG_CHUNK_SHIFT PAGE_SHIFT #define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT) #define ODEBUG_CHUNK_MASK (~(ODEBUG_CHUNK_SIZE - 1)) +/* + * We limit the freeing of debug objects via workqueue at a maximum + * frequency of 10Hz and about 1024 objects for each freeing operation. + * So it is freeing at most 10k debug objects per second. + */ +#define ODEBUG_FREE_WORK_MAX 1024 +#define ODEBUG_FREE_WORK_DELAY DIV_ROUND_UP(HZ, 10) + struct debug_bucket { struct hlist_head list; raw_spinlock_t lock; }; +/* + * Debug object percpu free list + * Access is protected by disabling irq + */ +struct debug_percpu_free { + struct hlist_head free_objs; + int obj_free; +}; + +static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool); + static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE]; static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata; @@ -44,13 +65,20 @@ static DEFINE_RAW_SPINLOCK(pool_lock); static HLIST_HEAD(obj_pool); static HLIST_HEAD(obj_to_free); +/* + * Because of the presence of percpu free pools, obj_pool_free will + * under-count those in the percpu free pools. Similarly, obj_pool_used + * will over-count those in the percpu free pools. Adjustments will be + * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used + * can be off. + */ static int obj_pool_min_free = ODEBUG_POOL_SIZE; static int obj_pool_free = ODEBUG_POOL_SIZE; static int obj_pool_used; static int obj_pool_max_used; +static bool obj_freeing; /* The number of objs on the global free list */ static int obj_nr_tofree; -static struct kmem_cache *obj_cache; static int debug_objects_maxchain __read_mostly; static int __maybe_unused debug_objects_maxchecked __read_mostly; @@ -63,6 +91,7 @@ static int debug_objects_pool_size __read_mostly static int debug_objects_pool_min_level __read_mostly = ODEBUG_POOL_MIN_LEVEL; static struct debug_obj_descr *descr_test __read_mostly; +static struct kmem_cache *obj_cache __read_mostly; /* * Track numbers of kmem_cache_alloc()/free() calls done. @@ -71,7 +100,7 @@ static int debug_objects_allocated; static int debug_objects_freed; static void free_obj_work(struct work_struct *work); -static DECLARE_WORK(debug_obj_work, free_obj_work); +static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work); static int __init enable_object_debug(char *str) { @@ -100,7 +129,7 @@ static const char *obj_states[ODEBUG_STATE_MAX] = { static void fill_pool(void) { gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN; - struct debug_obj *new, *obj; + struct debug_obj *obj; unsigned long flags; if (likely(obj_pool_free >= debug_objects_pool_min_level)) @@ -116,7 +145,7 @@ static void fill_pool(void) * Recheck with the lock held as the worker thread might have * won the race and freed the global free list already. */ - if (obj_nr_tofree) { + while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) { obj = hlist_entry(obj_to_free.first, typeof(*obj), node); hlist_del(&obj->node); obj_nr_tofree--; @@ -130,15 +159,23 @@ static void fill_pool(void) return; while (obj_pool_free < debug_objects_pool_min_level) { + struct debug_obj *new[ODEBUG_BATCH_SIZE]; + int cnt; - new = kmem_cache_zalloc(obj_cache, gfp); - if (!new) + for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) { + new[cnt] = kmem_cache_zalloc(obj_cache, gfp); + if (!new[cnt]) + break; + } + if (!cnt) return; raw_spin_lock_irqsave(&pool_lock, flags); - hlist_add_head(&new->node, &obj_pool); - debug_objects_allocated++; - obj_pool_free++; + while (cnt) { + hlist_add_head(&new[--cnt]->node, &obj_pool); + debug_objects_allocated++; + obj_pool_free++; + } raw_spin_unlock_irqrestore(&pool_lock, flags); } } @@ -163,36 +200,81 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) } /* + * Allocate a new object from the hlist + */ +static struct debug_obj *__alloc_object(struct hlist_head *list) +{ + struct debug_obj *obj = NULL; + + if (list->first) { + obj = hlist_entry(list->first, typeof(*obj), node); + hlist_del(&obj->node); + } + + return obj; +} + +/* * Allocate a new object. If the pool is empty, switch off the debugger. * Must be called with interrupts disabled. */ static struct debug_obj * alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) { - struct debug_obj *obj = NULL; + struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool); + struct debug_obj *obj; - raw_spin_lock(&pool_lock); - if (obj_pool.first) { - obj = hlist_entry(obj_pool.first, typeof(*obj), node); + if (likely(obj_cache)) { + obj = __alloc_object(&percpu_pool->free_objs); + if (obj) { + percpu_pool->obj_free--; + goto init_obj; + } + } - obj->object = addr; - obj->descr = descr; - obj->state = ODEBUG_STATE_NONE; - obj->astate = 0; - hlist_del(&obj->node); + raw_spin_lock(&pool_lock); + obj = __alloc_object(&obj_pool); + if (obj) { + obj_pool_used++; + obj_pool_free--; - hlist_add_head(&obj->node, &b->list); + /* + * Looking ahead, allocate one batch of debug objects and + * put them into the percpu free pool. + */ + if (likely(obj_cache)) { + int i; + + for (i = 0; i < ODEBUG_BATCH_SIZE; i++) { + struct debug_obj *obj2; + + obj2 = __alloc_object(&obj_pool); + if (!obj2) + break; + hlist_add_head(&obj2->node, + &percpu_pool->free_objs); + percpu_pool->obj_free++; + obj_pool_used++; + obj_pool_free--; + } + } - obj_pool_used++; if (obj_pool_used > obj_pool_max_used) obj_pool_max_used = obj_pool_used; - obj_pool_free--; if (obj_pool_free < obj_pool_min_free) obj_pool_min_free = obj_pool_free; } raw_spin_unlock(&pool_lock); +init_obj: + if (obj) { + obj->object = addr; + obj->descr = descr; + obj->state = ODEBUG_STATE_NONE; + obj->astate = 0; + hlist_add_head(&obj->node, &b->list); + } return obj; } @@ -209,13 +291,19 @@ static void free_obj_work(struct work_struct *work) unsigned long flags; HLIST_HEAD(tofree); + WRITE_ONCE(obj_freeing, false); if (!raw_spin_trylock_irqsave(&pool_lock, flags)) return; + if (obj_pool_free >= debug_objects_pool_size) + goto free_objs; + /* * The objs on the pool list might be allocated before the work is * run, so recheck if pool list it full or not, if not fill pool - * list from the global free list + * list from the global free list. As it is likely that a workload + * may be gearing up to use more and more objects, don't free any + * of them until the next round. */ while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) { obj = hlist_entry(obj_to_free.first, typeof(*obj), node); @@ -224,7 +312,10 @@ static void free_obj_work(struct work_struct *work) obj_pool_free++; obj_nr_tofree--; } + raw_spin_unlock_irqrestore(&pool_lock, flags); + return; +free_objs: /* * Pool list is already full and there are still objs on the free * list. Move remaining free objs to a temporary list to free the @@ -243,24 +334,86 @@ static void free_obj_work(struct work_struct *work) } } -static bool __free_object(struct debug_obj *obj) +static void __free_object(struct debug_obj *obj) { + struct debug_obj *objs[ODEBUG_BATCH_SIZE]; + struct debug_percpu_free *percpu_pool; + int lookahead_count = 0; unsigned long flags; bool work; - raw_spin_lock_irqsave(&pool_lock, flags); - work = (obj_pool_free > debug_objects_pool_size) && obj_cache; + local_irq_save(flags); + if (!obj_cache) + goto free_to_obj_pool; + + /* + * Try to free it into the percpu pool first. + */ + percpu_pool = this_cpu_ptr(&percpu_obj_pool); + if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) { + hlist_add_head(&obj->node, &percpu_pool->free_objs); + percpu_pool->obj_free++; + local_irq_restore(flags); + return; + } + + /* + * As the percpu pool is full, look ahead and pull out a batch + * of objects from the percpu pool and free them as well. + */ + for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) { + objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs); + if (!objs[lookahead_count]) + break; + percpu_pool->obj_free--; + } + +free_to_obj_pool: + raw_spin_lock(&pool_lock); + work = (obj_pool_free > debug_objects_pool_size) && obj_cache && + (obj_nr_tofree < ODEBUG_FREE_WORK_MAX); obj_pool_used--; if (work) { obj_nr_tofree++; hlist_add_head(&obj->node, &obj_to_free); + if (lookahead_count) { + obj_nr_tofree += lookahead_count; + obj_pool_used -= lookahead_count; + while (lookahead_count) { + hlist_add_head(&objs[--lookahead_count]->node, + &obj_to_free); + } + } + + if ((obj_pool_free > debug_objects_pool_size) && + (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) { + int i; + + /* + * Free one more batch of objects from obj_pool. + */ + for (i = 0; i < ODEBUG_BATCH_SIZE; i++) { + obj = __alloc_object(&obj_pool); + hlist_add_head(&obj->node, &obj_to_free); + obj_pool_free--; + obj_nr_tofree++; + } + } } else { obj_pool_free++; hlist_add_head(&obj->node, &obj_pool); + if (lookahead_count) { + obj_pool_free += lookahead_count; + obj_pool_used -= lookahead_count; + while (lookahead_count) { + hlist_add_head(&objs[--lookahead_count]->node, + &obj_pool); + } + } } - raw_spin_unlock_irqrestore(&pool_lock, flags); - return work; + raw_spin_unlock(&pool_lock); + local_irq_restore(flags); } /* @@ -269,8 +422,11 @@ static bool __free_object(struct debug_obj *obj) */ static void free_object(struct debug_obj *obj) { - if (__free_object(obj)) - schedule_work(&debug_obj_work); + __free_object(obj); + if (!obj_freeing && obj_nr_tofree) { + WRITE_ONCE(obj_freeing, true); + schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY); + } } /* @@ -372,6 +528,7 @@ static void __debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack) { enum debug_obj_state state; + bool check_stack = false; struct debug_bucket *db; struct debug_obj *obj; unsigned long flags; @@ -391,7 +548,7 @@ __debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack) debug_objects_oom(); return; } - debug_object_is_on_stack(addr, onstack); + check_stack = true; } switch (obj->state) { @@ -402,20 +559,23 @@ __debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack) break; case ODEBUG_STATE_ACTIVE: - debug_print_object(obj, "init"); state = obj->state; raw_spin_unlock_irqrestore(&db->lock, flags); + debug_print_object(obj, "init"); debug_object_fixup(descr->fixup_init, addr, state); return; case ODEBUG_STATE_DESTROYED: + raw_spin_unlock_irqrestore(&db->lock, flags); debug_print_object(obj, "init"); - break; + return; default: break; } raw_spin_unlock_irqrestore(&db->lock, flags); + if (check_stack) + debug_object_is_on_stack(addr, onstack); } /** @@ -473,6 +633,8 @@ int debug_object_activate(void *addr, struct debug_obj_descr *descr) obj = lookup_object(addr, db); if (obj) { + bool print_object = false; + switch (obj->state) { case ODEBUG_STATE_INIT: case ODEBUG_STATE_INACTIVE: @@ -481,14 +643,14 @@ int debug_object_activate(void *addr, struct debug_obj_descr *descr) break; case ODEBUG_STATE_ACTIVE: - debug_print_object(obj, "activate"); state = obj->state; raw_spin_unlock_irqrestore(&db->lock, flags); + debug_print_object(obj, "activate"); ret = debug_object_fixup(descr->fixup_activate, addr, state); return ret ? 0 : -EINVAL; case ODEBUG_STATE_DESTROYED: - debug_print_object(obj, "activate"); + print_object = true; ret = -EINVAL; break; default: @@ -496,10 +658,13 @@ int debug_object_activate(void *addr, struct debug_obj_descr *descr) break; } raw_spin_unlock_irqrestore(&db->lock, flags); + if (print_object) + debug_print_object(obj, "activate"); return ret; } raw_spin_unlock_irqrestore(&db->lock, flags); + /* * We are here when a static object is activated. We * let the type specific code confirm whether this is @@ -531,6 +696,7 @@ void debug_object_deactivate(void *addr, struct debug_obj_descr *descr) struct debug_bucket *db; struct debug_obj *obj; unsigned long flags; + bool print_object = false; if (!debug_objects_enabled) return; @@ -548,24 +714,27 @@ void debug_object_deactivate(void *addr, struct debug_obj_descr *descr) if (!obj->astate) obj->state = ODEBUG_STATE_INACTIVE; else - debug_print_object(obj, "deactivate"); + print_object = true; break; case ODEBUG_STATE_DESTROYED: - debug_print_object(obj, "deactivate"); + print_object = true; break; default: break; } - } else { + } + + raw_spin_unlock_irqrestore(&db->lock, flags); + if (!obj) { struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr }; debug_print_object(&o, "deactivate"); + } else if (print_object) { + debug_print_object(obj, "deactivate"); } - - raw_spin_unlock_irqrestore(&db->lock, flags); } EXPORT_SYMBOL_GPL(debug_object_deactivate); @@ -580,6 +749,7 @@ void debug_object_destroy(void *addr, struct debug_obj_descr *descr) struct debug_bucket *db; struct debug_obj *obj; unsigned long flags; + bool print_object = false; if (!debug_objects_enabled) return; @@ -599,20 +769,22 @@ void debug_object_destroy(void *addr, struct debug_obj_descr *descr) obj->state = ODEBUG_STATE_DESTROYED; break; case ODEBUG_STATE_ACTIVE: - debug_print_object(obj, "destroy"); state = obj->state; raw_spin_unlock_irqrestore(&db->lock, flags); + debug_print_object(obj, "destroy"); debug_object_fixup(descr->fixup_destroy, addr, state); return; case ODEBUG_STATE_DESTROYED: - debug_print_object(obj, "destroy"); + print_object = true; break; default: break; } out_unlock: raw_spin_unlock_irqrestore(&db->lock, flags); + if (print_object) + debug_print_object(obj, "destroy"); } EXPORT_SYMBOL_GPL(debug_object_destroy); @@ -641,9 +813,9 @@ void debug_object_free(void *addr, struct debug_obj_descr *descr) switch (obj->state) { case ODEBUG_STATE_ACTIVE: - debug_print_object(obj, "free"); state = obj->state; raw_spin_unlock_irqrestore(&db->lock, flags); + debug_print_object(obj, "free"); debug_object_fixup(descr->fixup_free, addr, state); return; default: @@ -716,6 +888,7 @@ debug_object_active_state(void *addr, struct debug_obj_descr *descr, struct debug_bucket *db; struct debug_obj *obj; unsigned long flags; + bool print_object = false; if (!debug_objects_enabled) return; @@ -731,22 +904,25 @@ debug_object_active_state(void *addr, struct debug_obj_descr *descr, if (obj->astate == expect) obj->astate = next; else - debug_print_object(obj, "active_state"); + print_object = true; break; default: - debug_print_object(obj, "active_state"); + print_object = true; break; } - } else { + } + + raw_spin_unlock_irqrestore(&db->lock, flags); + if (!obj) { struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr }; debug_print_object(&o, "active_state"); + } else if (print_object) { + debug_print_object(obj, "active_state"); } - - raw_spin_unlock_irqrestore(&db->lock, flags); } EXPORT_SYMBOL_GPL(debug_object_active_state); @@ -760,7 +936,6 @@ static void __debug_check_no_obj_freed(const void *address, unsigned long size) struct hlist_node *tmp; struct debug_obj *obj; int cnt, objs_checked = 0; - bool work = false; saddr = (unsigned long) address; eaddr = saddr + size; @@ -782,16 +957,16 @@ repeat: switch (obj->state) { case ODEBUG_STATE_ACTIVE: - debug_print_object(obj, "free"); descr = obj->descr; state = obj->state; raw_spin_unlock_irqrestore(&db->lock, flags); + debug_print_object(obj, "free"); debug_object_fixup(descr->fixup_free, (void *) oaddr, state); goto repeat; default: hlist_del(&obj->node); - work |= __free_object(obj); + __free_object(obj); break; } } @@ -807,8 +982,10 @@ repeat: debug_objects_maxchecked = objs_checked; /* Schedule work to actually kmem_cache_free() objects */ - if (work) - schedule_work(&debug_obj_work); + if (!obj_freeing && obj_nr_tofree) { + WRITE_ONCE(obj_freeing, true); + schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY); + } } void debug_check_no_obj_freed(const void *address, unsigned long size) @@ -822,13 +999,19 @@ void debug_check_no_obj_freed(const void *address, unsigned long size) static int debug_stats_show(struct seq_file *m, void *v) { + int cpu, obj_percpu_free = 0; + + for_each_possible_cpu(cpu) + obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu); + seq_printf(m, "max_chain :%d\n", debug_objects_maxchain); seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked); seq_printf(m, "warnings :%d\n", debug_objects_warnings); seq_printf(m, "fixups :%d\n", debug_objects_fixups); - seq_printf(m, "pool_free :%d\n", obj_pool_free); + seq_printf(m, "pool_free :%d\n", obj_pool_free + obj_percpu_free); + seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free); seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free); - seq_printf(m, "pool_used :%d\n", obj_pool_used); + seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free); seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used); seq_printf(m, "on_free_list :%d\n", obj_nr_tofree); seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated); @@ -850,26 +1033,16 @@ static const struct file_operations debug_stats_fops = { static int __init debug_objects_init_debugfs(void) { - struct dentry *dbgdir, *dbgstats; + struct dentry *dbgdir; if (!debug_objects_enabled) return 0; dbgdir = debugfs_create_dir("debug_objects", NULL); - if (!dbgdir) - return -ENOMEM; - dbgstats = debugfs_create_file("stats", 0444, dbgdir, NULL, - &debug_stats_fops); - if (!dbgstats) - goto err; + debugfs_create_file("stats", 0444, dbgdir, NULL, &debug_stats_fops); return 0; - -err: - debugfs_remove(dbgdir); - - return -ENOMEM; } __initcall(debug_objects_init_debugfs); @@ -1175,9 +1348,20 @@ free: */ void __init debug_objects_mem_init(void) { + int cpu, extras; + if (!debug_objects_enabled) return; + /* + * Initialize the percpu object pools + * + * Initialization is not strictly necessary, but was done for + * completeness. + */ + for_each_possible_cpu(cpu) + INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu)); + obj_cache = kmem_cache_create("debug_objects_cache", sizeof (struct debug_obj), 0, SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE, @@ -1194,6 +1378,7 @@ void __init debug_objects_mem_init(void) * Increase the thresholds for allocating and freeing objects * according to the number of possible CPUs available in the system. */ - debug_objects_pool_size += num_possible_cpus() * 32; - debug_objects_pool_min_level += num_possible_cpus() * 4; + extras = num_possible_cpus() * ODEBUG_BATCH_SIZE; + debug_objects_pool_size += extras; + debug_objects_pool_min_level += extras; } diff --git a/lib/devres.c b/lib/devres.c index 69bed2f38306..6a0e9bd6524a 100644 --- a/lib/devres.c +++ b/lib/devres.c @@ -131,7 +131,8 @@ EXPORT_SYMBOL(devm_iounmap); * if (IS_ERR(base)) * return PTR_ERR(base); */ -void __iomem *devm_ioremap_resource(struct device *dev, struct resource *res) +void __iomem *devm_ioremap_resource(struct device *dev, + const struct resource *res) { resource_size_t size; void __iomem *dest_ptr; diff --git a/lib/digsig.c b/lib/digsig.c index 3cf89c775ab2..e0627c3e53b2 100644 --- a/lib/digsig.c +++ b/lib/digsig.c @@ -218,7 +218,7 @@ int digsig_verify(struct key *keyring, const char *sig, int siglen, /* search in specific keyring */ key_ref_t kref; kref = keyring_search(make_key_ref(keyring, 1UL), - &key_type_user, name); + &key_type_user, name, true); if (IS_ERR(kref)) key = ERR_CAST(kref); else diff --git a/lib/dim/Makefile b/lib/dim/Makefile new file mode 100644 index 000000000000..1d6858a108cb --- /dev/null +++ b/lib/dim/Makefile @@ -0,0 +1,7 @@ +# +# DIM Dynamic Interrupt Moderation library +# + +obj-$(CONFIG_DIMLIB) += dim.o + +dim-y := dim.o net_dim.o rdma_dim.o diff --git a/lib/dim/dim.c b/lib/dim/dim.c new file mode 100644 index 000000000000..38045d6d0538 --- /dev/null +++ b/lib/dim/dim.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved. + */ + +#include <linux/dim.h> + +bool dim_on_top(struct dim *dim) +{ + switch (dim->tune_state) { + case DIM_PARKING_ON_TOP: + case DIM_PARKING_TIRED: + return true; + case DIM_GOING_RIGHT: + return (dim->steps_left > 1) && (dim->steps_right == 1); + default: /* DIM_GOING_LEFT */ + return (dim->steps_right > 1) && (dim->steps_left == 1); + } +} +EXPORT_SYMBOL(dim_on_top); + +void dim_turn(struct dim *dim) +{ + switch (dim->tune_state) { + case DIM_PARKING_ON_TOP: + case DIM_PARKING_TIRED: + break; + case DIM_GOING_RIGHT: + dim->tune_state = DIM_GOING_LEFT; + dim->steps_left = 0; + break; + case DIM_GOING_LEFT: + dim->tune_state = DIM_GOING_RIGHT; + dim->steps_right = 0; + break; + } +} +EXPORT_SYMBOL(dim_turn); + +void dim_park_on_top(struct dim *dim) +{ + dim->steps_right = 0; + dim->steps_left = 0; + dim->tired = 0; + dim->tune_state = DIM_PARKING_ON_TOP; +} +EXPORT_SYMBOL(dim_park_on_top); + +void dim_park_tired(struct dim *dim) +{ + dim->steps_right = 0; + dim->steps_left = 0; + dim->tune_state = DIM_PARKING_TIRED; +} +EXPORT_SYMBOL(dim_park_tired); + +void dim_calc_stats(struct dim_sample *start, struct dim_sample *end, + struct dim_stats *curr_stats) +{ + /* u32 holds up to 71 minutes, should be enough */ + u32 delta_us = ktime_us_delta(end->time, start->time); + u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr); + u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr, + start->byte_ctr); + u32 ncomps = BIT_GAP(BITS_PER_TYPE(u32), end->comp_ctr, + start->comp_ctr); + + if (!delta_us) + return; + + curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); + curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); + curr_stats->epms = DIV_ROUND_UP(DIM_NEVENTS * USEC_PER_MSEC, + delta_us); + curr_stats->cpms = DIV_ROUND_UP(ncomps * USEC_PER_MSEC, delta_us); + if (curr_stats->epms != 0) + curr_stats->cpe_ratio = DIV_ROUND_DOWN_ULL( + curr_stats->cpms * 100, curr_stats->epms); + else + curr_stats->cpe_ratio = 0; + +} +EXPORT_SYMBOL(dim_calc_stats); diff --git a/lib/dim/net_dim.c b/lib/dim/net_dim.c new file mode 100644 index 000000000000..a4db51c21266 --- /dev/null +++ b/lib/dim/net_dim.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. + */ + +#include <linux/dim.h> + +/* + * Net DIM profiles: + * There are different set of profiles for each CQ period mode. + * There are different set of profiles for RX/TX CQs. + * Each profile size must be of NET_DIM_PARAMS_NUM_PROFILES + */ +#define NET_DIM_PARAMS_NUM_PROFILES 5 +#define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 +#define NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE 128 +#define NET_DIM_DEF_PROFILE_CQE 1 +#define NET_DIM_DEF_PROFILE_EQE 1 + +#define NET_DIM_RX_EQE_PROFILES { \ + {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ +} + +#define NET_DIM_RX_CQE_PROFILES { \ + {2, 256}, \ + {8, 128}, \ + {16, 64}, \ + {32, 64}, \ + {64, 64} \ +} + +#define NET_DIM_TX_EQE_PROFILES { \ + {1, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {8, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {32, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {64, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {128, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE} \ +} + +#define NET_DIM_TX_CQE_PROFILES { \ + {5, 128}, \ + {8, 64}, \ + {16, 32}, \ + {32, 32}, \ + {64, 32} \ +} + +static const struct dim_cq_moder +rx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { + NET_DIM_RX_EQE_PROFILES, + NET_DIM_RX_CQE_PROFILES, +}; + +static const struct dim_cq_moder +tx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { + NET_DIM_TX_EQE_PROFILES, + NET_DIM_TX_CQE_PROFILES, +}; + +struct dim_cq_moder +net_dim_get_rx_moderation(u8 cq_period_mode, int ix) +{ + struct dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix]; + + cq_moder.cq_period_mode = cq_period_mode; + return cq_moder; +} +EXPORT_SYMBOL(net_dim_get_rx_moderation); + +struct dim_cq_moder +net_dim_get_def_rx_moderation(u8 cq_period_mode) +{ + u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? + NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE; + + return net_dim_get_rx_moderation(cq_period_mode, profile_ix); +} +EXPORT_SYMBOL(net_dim_get_def_rx_moderation); + +struct dim_cq_moder +net_dim_get_tx_moderation(u8 cq_period_mode, int ix) +{ + struct dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix]; + + cq_moder.cq_period_mode = cq_period_mode; + return cq_moder; +} +EXPORT_SYMBOL(net_dim_get_tx_moderation); + +struct dim_cq_moder +net_dim_get_def_tx_moderation(u8 cq_period_mode) +{ + u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? + NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE; + + return net_dim_get_tx_moderation(cq_period_mode, profile_ix); +} +EXPORT_SYMBOL(net_dim_get_def_tx_moderation); + +static int net_dim_step(struct dim *dim) +{ + if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) + return DIM_TOO_TIRED; + + switch (dim->tune_state) { + case DIM_PARKING_ON_TOP: + case DIM_PARKING_TIRED: + break; + case DIM_GOING_RIGHT: + if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1)) + return DIM_ON_EDGE; + dim->profile_ix++; + dim->steps_right++; + break; + case DIM_GOING_LEFT: + if (dim->profile_ix == 0) + return DIM_ON_EDGE; + dim->profile_ix--; + dim->steps_left++; + break; + } + + dim->tired++; + return DIM_STEPPED; +} + +static void net_dim_exit_parking(struct dim *dim) +{ + dim->tune_state = dim->profile_ix ? DIM_GOING_LEFT : DIM_GOING_RIGHT; + net_dim_step(dim); +} + +static int net_dim_stats_compare(struct dim_stats *curr, + struct dim_stats *prev) +{ + if (!prev->bpms) + return curr->bpms ? DIM_STATS_BETTER : DIM_STATS_SAME; + + if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) + return (curr->bpms > prev->bpms) ? DIM_STATS_BETTER : + DIM_STATS_WORSE; + + if (!prev->ppms) + return curr->ppms ? DIM_STATS_BETTER : + DIM_STATS_SAME; + + if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) + return (curr->ppms > prev->ppms) ? DIM_STATS_BETTER : + DIM_STATS_WORSE; + + if (!prev->epms) + return DIM_STATS_SAME; + + if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) + return (curr->epms < prev->epms) ? DIM_STATS_BETTER : + DIM_STATS_WORSE; + + return DIM_STATS_SAME; +} + +static bool net_dim_decision(struct dim_stats *curr_stats, struct dim *dim) +{ + int prev_state = dim->tune_state; + int prev_ix = dim->profile_ix; + int stats_res; + int step_res; + + switch (dim->tune_state) { + case DIM_PARKING_ON_TOP: + stats_res = net_dim_stats_compare(curr_stats, + &dim->prev_stats); + if (stats_res != DIM_STATS_SAME) + net_dim_exit_parking(dim); + break; + + case DIM_PARKING_TIRED: + dim->tired--; + if (!dim->tired) + net_dim_exit_parking(dim); + break; + + case DIM_GOING_RIGHT: + case DIM_GOING_LEFT: + stats_res = net_dim_stats_compare(curr_stats, + &dim->prev_stats); + if (stats_res != DIM_STATS_BETTER) + dim_turn(dim); + + if (dim_on_top(dim)) { + dim_park_on_top(dim); + break; + } + + step_res = net_dim_step(dim); + switch (step_res) { + case DIM_ON_EDGE: + dim_park_on_top(dim); + break; + case DIM_TOO_TIRED: + dim_park_tired(dim); + break; + } + + break; + } + + if (prev_state != DIM_PARKING_ON_TOP || + dim->tune_state != DIM_PARKING_ON_TOP) + dim->prev_stats = *curr_stats; + + return dim->profile_ix != prev_ix; +} + +void net_dim(struct dim *dim, struct dim_sample end_sample) +{ + struct dim_stats curr_stats; + u16 nevents; + + switch (dim->state) { + case DIM_MEASURE_IN_PROGRESS: + nevents = BIT_GAP(BITS_PER_TYPE(u16), + end_sample.event_ctr, + dim->start_sample.event_ctr); + if (nevents < DIM_NEVENTS) + break; + dim_calc_stats(&dim->start_sample, &end_sample, &curr_stats); + if (net_dim_decision(&curr_stats, dim)) { + dim->state = DIM_APPLY_NEW_PROFILE; + schedule_work(&dim->work); + break; + } + /* fall through */ + case DIM_START_MEASURE: + dim_update_sample(end_sample.event_ctr, end_sample.pkt_ctr, + end_sample.byte_ctr, &dim->start_sample); + dim->state = DIM_MEASURE_IN_PROGRESS; + break; + case DIM_APPLY_NEW_PROFILE: + break; + } +} +EXPORT_SYMBOL(net_dim); diff --git a/lib/dim/rdma_dim.c b/lib/dim/rdma_dim.c new file mode 100644 index 000000000000..f7e26c7b4749 --- /dev/null +++ b/lib/dim/rdma_dim.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved. + */ + +#include <linux/dim.h> + +static int rdma_dim_step(struct dim *dim) +{ + if (dim->tune_state == DIM_GOING_RIGHT) { + if (dim->profile_ix == (RDMA_DIM_PARAMS_NUM_PROFILES - 1)) + return DIM_ON_EDGE; + dim->profile_ix++; + dim->steps_right++; + } + if (dim->tune_state == DIM_GOING_LEFT) { + if (dim->profile_ix == 0) + return DIM_ON_EDGE; + dim->profile_ix--; + dim->steps_left++; + } + + return DIM_STEPPED; +} + +static int rdma_dim_stats_compare(struct dim_stats *curr, + struct dim_stats *prev) +{ + /* first stat */ + if (!prev->cpms) + return DIM_STATS_SAME; + + if (IS_SIGNIFICANT_DIFF(curr->cpms, prev->cpms)) + return (curr->cpms > prev->cpms) ? DIM_STATS_BETTER : + DIM_STATS_WORSE; + + if (IS_SIGNIFICANT_DIFF(curr->cpe_ratio, prev->cpe_ratio)) + return (curr->cpe_ratio > prev->cpe_ratio) ? DIM_STATS_BETTER : + DIM_STATS_WORSE; + + return DIM_STATS_SAME; +} + +static bool rdma_dim_decision(struct dim_stats *curr_stats, struct dim *dim) +{ + int prev_ix = dim->profile_ix; + u8 state = dim->tune_state; + int stats_res; + int step_res; + + if (state != DIM_PARKING_ON_TOP && state != DIM_PARKING_TIRED) { + stats_res = rdma_dim_stats_compare(curr_stats, + &dim->prev_stats); + + switch (stats_res) { + case DIM_STATS_SAME: + if (curr_stats->cpe_ratio <= 50 * prev_ix) + dim->profile_ix = 0; + break; + case DIM_STATS_WORSE: + dim_turn(dim); + /* fall through */ + case DIM_STATS_BETTER: + step_res = rdma_dim_step(dim); + if (step_res == DIM_ON_EDGE) + dim_turn(dim); + break; + } + } + + dim->prev_stats = *curr_stats; + + return dim->profile_ix != prev_ix; +} + +void rdma_dim(struct dim *dim, u64 completions) +{ + struct dim_sample *curr_sample = &dim->measuring_sample; + struct dim_stats curr_stats; + u32 nevents; + + dim_update_sample_with_comps(curr_sample->event_ctr + 1, 0, 0, + curr_sample->comp_ctr + completions, + &dim->measuring_sample); + + switch (dim->state) { + case DIM_MEASURE_IN_PROGRESS: + nevents = curr_sample->event_ctr - dim->start_sample.event_ctr; + if (nevents < DIM_NEVENTS) + break; + dim_calc_stats(&dim->start_sample, curr_sample, &curr_stats); + if (rdma_dim_decision(&curr_stats, dim)) { + dim->state = DIM_APPLY_NEW_PROFILE; + schedule_work(&dim->work); + break; + } + /* fall through */ + case DIM_START_MEASURE: + dim->state = DIM_MEASURE_IN_PROGRESS; + dim_update_sample_with_comps(curr_sample->event_ctr, 0, 0, + curr_sample->comp_ctr, + &dim->start_sample); + break; + case DIM_APPLY_NEW_PROFILE: + break; + } +} +EXPORT_SYMBOL(rdma_dim); diff --git a/lib/dump_stack.c b/lib/dump_stack.c index 5cff72f18c4a..33ffbf308853 100644 --- a/lib/dump_stack.c +++ b/lib/dump_stack.c @@ -106,7 +106,12 @@ retry: was_locked = 1; } else { local_irq_restore(flags); - cpu_relax(); + /* + * Wait for the lock to release before jumping to + * atomic_cmpxchg() in order to mitigate the thundering herd + * problem. + */ + do { cpu_relax(); } while (atomic_read(&dump_lock) != -1); goto retry; } diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index 8a16c2d498e9..c60409138e13 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -993,20 +993,14 @@ static __initdata int ddebug_init_success; static int __init dynamic_debug_init_debugfs(void) { - struct dentry *dir, *file; + struct dentry *dir; if (!ddebug_init_success) return -ENODEV; dir = debugfs_create_dir("dynamic_debug", NULL); - if (!dir) - return -ENOMEM; - file = debugfs_create_file("control", 0644, dir, NULL, - &ddebug_proc_fops); - if (!file) { - debugfs_remove(dir); - return -ENOMEM; - } + debugfs_create_file("control", 0644, dir, NULL, &ddebug_proc_fops); + return 0; } diff --git a/lib/extable.c b/lib/extable.c index 25da4071122a..c3e59caf7ffa 100644 --- a/lib/extable.c +++ b/lib/extable.c @@ -10,6 +10,7 @@ #include <linux/init.h> #include <linux/sort.h> #include <linux/uaccess.h> +#include <linux/extable.h> #ifndef ARCH_HAS_RELATIVE_EXTABLE #define ex_to_insn(x) ((x)->insn) diff --git a/lib/fault-inject.c b/lib/fault-inject.c index 3cb21b2bf088..8186ca84910b 100644 --- a/lib/fault-inject.c +++ b/lib/fault-inject.c @@ -166,10 +166,10 @@ static int debugfs_ul_get(void *data, u64 *val) DEFINE_SIMPLE_ATTRIBUTE(fops_ul, debugfs_ul_get, debugfs_ul_set, "%llu\n"); -static struct dentry *debugfs_create_ul(const char *name, umode_t mode, - struct dentry *parent, unsigned long *value) +static void debugfs_create_ul(const char *name, umode_t mode, + struct dentry *parent, unsigned long *value) { - return debugfs_create_file(name, mode, parent, value, &fops_ul); + debugfs_create_file(name, mode, parent, value, &fops_ul); } #ifdef CONFIG_FAULT_INJECTION_STACKTRACE_FILTER @@ -185,12 +185,11 @@ static int debugfs_stacktrace_depth_set(void *data, u64 val) DEFINE_SIMPLE_ATTRIBUTE(fops_stacktrace_depth, debugfs_ul_get, debugfs_stacktrace_depth_set, "%llu\n"); -static struct dentry *debugfs_create_stacktrace_depth( - const char *name, umode_t mode, - struct dentry *parent, unsigned long *value) +static void debugfs_create_stacktrace_depth(const char *name, umode_t mode, + struct dentry *parent, + unsigned long *value) { - return debugfs_create_file(name, mode, parent, value, - &fops_stacktrace_depth); + debugfs_create_file(name, mode, parent, value, &fops_stacktrace_depth); } #endif /* CONFIG_FAULT_INJECTION_STACKTRACE_FILTER */ @@ -202,51 +201,31 @@ struct dentry *fault_create_debugfs_attr(const char *name, struct dentry *dir; dir = debugfs_create_dir(name, parent); - if (!dir) - return ERR_PTR(-ENOMEM); - - if (!debugfs_create_ul("probability", mode, dir, &attr->probability)) - goto fail; - if (!debugfs_create_ul("interval", mode, dir, &attr->interval)) - goto fail; - if (!debugfs_create_atomic_t("times", mode, dir, &attr->times)) - goto fail; - if (!debugfs_create_atomic_t("space", mode, dir, &attr->space)) - goto fail; - if (!debugfs_create_ul("verbose", mode, dir, &attr->verbose)) - goto fail; - if (!debugfs_create_u32("verbose_ratelimit_interval_ms", mode, dir, - &attr->ratelimit_state.interval)) - goto fail; - if (!debugfs_create_u32("verbose_ratelimit_burst", mode, dir, - &attr->ratelimit_state.burst)) - goto fail; - if (!debugfs_create_bool("task-filter", mode, dir, &attr->task_filter)) - goto fail; + if (IS_ERR(dir)) + return dir; + + debugfs_create_ul("probability", mode, dir, &attr->probability); + debugfs_create_ul("interval", mode, dir, &attr->interval); + debugfs_create_atomic_t("times", mode, dir, &attr->times); + debugfs_create_atomic_t("space", mode, dir, &attr->space); + debugfs_create_ul("verbose", mode, dir, &attr->verbose); + debugfs_create_u32("verbose_ratelimit_interval_ms", mode, dir, + &attr->ratelimit_state.interval); + debugfs_create_u32("verbose_ratelimit_burst", mode, dir, + &attr->ratelimit_state.burst); + debugfs_create_bool("task-filter", mode, dir, &attr->task_filter); #ifdef CONFIG_FAULT_INJECTION_STACKTRACE_FILTER - - if (!debugfs_create_stacktrace_depth("stacktrace-depth", mode, dir, - &attr->stacktrace_depth)) - goto fail; - if (!debugfs_create_ul("require-start", mode, dir, - &attr->require_start)) - goto fail; - if (!debugfs_create_ul("require-end", mode, dir, &attr->require_end)) - goto fail; - if (!debugfs_create_ul("reject-start", mode, dir, &attr->reject_start)) - goto fail; - if (!debugfs_create_ul("reject-end", mode, dir, &attr->reject_end)) - goto fail; - + debugfs_create_stacktrace_depth("stacktrace-depth", mode, dir, + &attr->stacktrace_depth); + debugfs_create_ul("require-start", mode, dir, &attr->require_start); + debugfs_create_ul("require-end", mode, dir, &attr->require_end); + debugfs_create_ul("reject-start", mode, dir, &attr->reject_start); + debugfs_create_ul("reject-end", mode, dir, &attr->reject_end); #endif /* CONFIG_FAULT_INJECTION_STACKTRACE_FILTER */ attr->dname = dget(dir); return dir; -fail: - debugfs_remove_recursive(dir); - - return ERR_PTR(-ENOMEM); } EXPORT_SYMBOL_GPL(fault_create_debugfs_attr); diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c index 9969358a7af5..e7258d8c252b 100644 --- a/lib/fonts/fonts.c +++ b/lib/fonts/fonts.c @@ -20,56 +20,42 @@ #endif #include <linux/font.h> -#define NO_FONTS - static const struct font_desc *fonts[] = { #ifdef CONFIG_FONT_8x8 -#undef NO_FONTS - &font_vga_8x8, + &font_vga_8x8, #endif #ifdef CONFIG_FONT_8x16 -#undef NO_FONTS - &font_vga_8x16, + &font_vga_8x16, #endif #ifdef CONFIG_FONT_6x11 -#undef NO_FONTS - &font_vga_6x11, + &font_vga_6x11, #endif #ifdef CONFIG_FONT_7x14 -#undef NO_FONTS - &font_7x14, + &font_7x14, #endif #ifdef CONFIG_FONT_SUN8x16 -#undef NO_FONTS - &font_sun_8x16, + &font_sun_8x16, #endif #ifdef CONFIG_FONT_SUN12x22 -#undef NO_FONTS - &font_sun_12x22, + &font_sun_12x22, #endif #ifdef CONFIG_FONT_10x18 -#undef NO_FONTS - &font_10x18, + &font_10x18, #endif #ifdef CONFIG_FONT_ACORN_8x8 -#undef NO_FONTS - &font_acorn_8x8, + &font_acorn_8x8, #endif #ifdef CONFIG_FONT_PEARL_8x8 -#undef NO_FONTS - &font_pearl_8x8, + &font_pearl_8x8, #endif #ifdef CONFIG_FONT_MINI_4x6 -#undef NO_FONTS - &font_mini_4x6, + &font_mini_4x6, #endif #ifdef CONFIG_FONT_6x10 -#undef NO_FONTS - &font_6x10, + &font_6x10, #endif #ifdef CONFIG_FONT_TER16x32 -#undef NO_FONTS - &font_ter_16x32, + &font_ter_16x32, #endif }; @@ -90,16 +76,17 @@ static const struct font_desc *fonts[] = { * specified font. * */ - const struct font_desc *find_font(const char *name) { - unsigned int i; + unsigned int i; - for (i = 0; i < num_fonts; i++) - if (!strcmp(fonts[i]->name, name)) - return fonts[i]; - return NULL; + BUILD_BUG_ON(!num_fonts); + for (i = 0; i < num_fonts; i++) + if (!strcmp(fonts[i]->name, name)) + return fonts[i]; + return NULL; } +EXPORT_SYMBOL(find_font); /** @@ -116,44 +103,46 @@ const struct font_desc *find_font(const char *name) * chosen font. * */ - const struct font_desc *get_default_font(int xres, int yres, u32 font_w, u32 font_h) { - int i, c, cc; - const struct font_desc *f, *g; - - g = NULL; - cc = -10000; - for(i=0; i<num_fonts; i++) { - f = fonts[i]; - c = f->pref; + int i, c, cc, res; + const struct font_desc *f, *g; + + g = NULL; + cc = -10000; + for (i = 0; i < num_fonts; i++) { + f = fonts[i]; + c = f->pref; #if defined(__mc68000__) #ifdef CONFIG_FONT_PEARL_8x8 - if (MACH_IS_AMIGA && f->idx == PEARL8x8_IDX) - c = 100; + if (MACH_IS_AMIGA && f->idx == PEARL8x8_IDX) + c = 100; #endif #ifdef CONFIG_FONT_6x11 - if (MACH_IS_MAC && xres < 640 && f->idx == VGA6x11_IDX) - c = 100; + if (MACH_IS_MAC && xres < 640 && f->idx == VGA6x11_IDX) + c = 100; #endif #endif - if ((yres < 400) == (f->height <= 8)) - c += 1000; + if ((yres < 400) == (f->height <= 8)) + c += 1000; + + /* prefer a bigger font for high resolution */ + res = (xres / f->width) * (yres / f->height) / 1000; + if (res > 20) + c += 20 - res; - if ((font_w & (1 << (f->width - 1))) && - (font_h & (1 << (f->height - 1)))) - c += 1000; + if ((font_w & (1 << (f->width - 1))) && + (font_h & (1 << (f->height - 1)))) + c += 1000; - if (c > cc) { - cc = c; - g = f; + if (c > cc) { + cc = c; + g = f; + } } - } - return g; + return g; } - -EXPORT_SYMBOL(find_font); EXPORT_SYMBOL(get_default_font); MODULE_AUTHOR("James Simmons <jsimmons@users.sf.net>"); diff --git a/lib/genalloc.c b/lib/genalloc.c index 5257f74fccf3..9fc31292cfa1 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -327,21 +327,45 @@ EXPORT_SYMBOL(gen_pool_alloc_algo_owner); * gen_pool_dma_alloc - allocate special memory from the pool for DMA usage * @pool: pool to allocate from * @size: number of bytes to allocate from the pool - * @dma: dma-view physical address return value. Use NULL if unneeded. + * @dma: dma-view physical address return value. Use %NULL if unneeded. * * Allocate the requested number of bytes from the specified pool. * Uses the pool allocation function (with first-fit algorithm by default). * Can not be used in NMI handler on architectures without * NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated memory, or %NULL on failure */ void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) { + return gen_pool_dma_alloc_algo(pool, size, dma, pool->algo, pool->data); +} +EXPORT_SYMBOL(gen_pool_dma_alloc); + +/** + * gen_pool_dma_alloc_algo - allocate special memory from the pool for DMA + * usage with the given pool algorithm + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @algo: algorithm passed from caller + * @data: data passed to algorithm + * + * Allocate the requested number of bytes from the specified pool. Uses the + * given pool allocation function. Can not be used in NMI handler on + * architectures without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated memory, or %NULL on failure + */ +void *gen_pool_dma_alloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data) +{ unsigned long vaddr; if (!pool) return NULL; - vaddr = gen_pool_alloc(pool, size); + vaddr = gen_pool_alloc_algo(pool, size, algo, data); if (!vaddr) return NULL; @@ -350,7 +374,102 @@ void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) return (void *)vaddr; } -EXPORT_SYMBOL(gen_pool_dma_alloc); +EXPORT_SYMBOL(gen_pool_dma_alloc_algo); + +/** + * gen_pool_dma_alloc_align - allocate special memory from the pool for DMA + * usage with the given alignment + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @align: alignment in bytes for starting address + * + * Allocate the requested number bytes from the specified pool, with the given + * alignment restriction. Can not be used in NMI handler on architectures + * without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated memory, or %NULL on failure + */ +void *gen_pool_dma_alloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align) +{ + struct genpool_data_align data = { .align = align }; + + return gen_pool_dma_alloc_algo(pool, size, dma, + gen_pool_first_fit_align, &data); +} +EXPORT_SYMBOL(gen_pool_dma_alloc_align); + +/** + * gen_pool_dma_zalloc - allocate special zeroed memory from the pool for + * DMA usage + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: dma-view physical address return value. Use %NULL if unneeded. + * + * Allocate the requested number of zeroed bytes from the specified pool. + * Uses the pool allocation function (with first-fit algorithm by default). + * Can not be used in NMI handler on architectures without + * NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated zeroed memory, or %NULL on failure + */ +void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) +{ + return gen_pool_dma_zalloc_algo(pool, size, dma, pool->algo, pool->data); +} +EXPORT_SYMBOL(gen_pool_dma_zalloc); + +/** + * gen_pool_dma_zalloc_algo - allocate special zeroed memory from the pool for + * DMA usage with the given pool algorithm + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @algo: algorithm passed from caller + * @data: data passed to algorithm + * + * Allocate the requested number of zeroed bytes from the specified pool. Uses + * the given pool allocation function. Can not be used in NMI handler on + * architectures without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated zeroed memory, or %NULL on failure + */ +void *gen_pool_dma_zalloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data) +{ + void *vaddr = gen_pool_dma_alloc_algo(pool, size, dma, algo, data); + + if (vaddr) + memset(vaddr, 0, size); + + return vaddr; +} +EXPORT_SYMBOL(gen_pool_dma_zalloc_algo); + +/** + * gen_pool_dma_zalloc_align - allocate special zeroed memory from the pool for + * DMA usage with the given alignment + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @align: alignment in bytes for starting address + * + * Allocate the requested number of zeroed bytes from the specified pool, + * with the given alignment restriction. Can not be used in NMI handler on + * architectures without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated zeroed memory, or %NULL on failure + */ +void *gen_pool_dma_zalloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align) +{ + struct genpool_data_align data = { .align = align }; + + return gen_pool_dma_zalloc_algo(pool, size, dma, + gen_pool_first_fit_align, &data); +} +EXPORT_SYMBOL(gen_pool_dma_zalloc_align); /** * gen_pool_free - free allocated special memory back to the pool diff --git a/lib/generic-radix-tree.c b/lib/generic-radix-tree.c index a7bafc413730..f25eb111c051 100644 --- a/lib/generic-radix-tree.c +++ b/lib/generic-radix-tree.c @@ -2,6 +2,7 @@ #include <linux/export.h> #include <linux/generic-radix-tree.h> #include <linux/gfp.h> +#include <linux/kmemleak.h> #define GENRADIX_ARY (PAGE_SIZE / sizeof(struct genradix_node *)) #define GENRADIX_ARY_SHIFT ilog2(GENRADIX_ARY) @@ -36,12 +37,12 @@ static inline size_t genradix_depth_size(unsigned depth) #define GENRADIX_DEPTH_MASK \ ((unsigned long) (roundup_pow_of_two(GENRADIX_MAX_DEPTH + 1) - 1)) -unsigned genradix_root_to_depth(struct genradix_root *r) +static inline unsigned genradix_root_to_depth(struct genradix_root *r) { return (unsigned long) r & GENRADIX_DEPTH_MASK; } -struct genradix_node *genradix_root_to_node(struct genradix_root *r) +static inline struct genradix_node *genradix_root_to_node(struct genradix_root *r) { return (void *) ((unsigned long) r & ~GENRADIX_DEPTH_MASK); } @@ -75,6 +76,27 @@ void *__genradix_ptr(struct __genradix *radix, size_t offset) } EXPORT_SYMBOL(__genradix_ptr); +static inline struct genradix_node *genradix_alloc_node(gfp_t gfp_mask) +{ + struct genradix_node *node; + + node = (struct genradix_node *)__get_free_page(gfp_mask|__GFP_ZERO); + + /* + * We're using pages (not slab allocations) directly for kernel data + * structures, so we need to explicitly inform kmemleak of them in order + * to avoid false positive memory leak reports. + */ + kmemleak_alloc(node, PAGE_SIZE, 1, gfp_mask); + return node; +} + +static inline void genradix_free_node(struct genradix_node *node) +{ + kmemleak_free(node); + free_page((unsigned long)node); +} + /* * Returns pointer to the specified byte @offset within @radix, allocating it if * necessary - newly allocated slots are always zeroed out: @@ -97,8 +119,7 @@ void *__genradix_ptr_alloc(struct __genradix *radix, size_t offset, break; if (!new_node) { - new_node = (void *) - __get_free_page(gfp_mask|__GFP_ZERO); + new_node = genradix_alloc_node(gfp_mask); if (!new_node) return NULL; } @@ -121,8 +142,7 @@ void *__genradix_ptr_alloc(struct __genradix *radix, size_t offset, n = READ_ONCE(*p); if (!n) { if (!new_node) { - new_node = (void *) - __get_free_page(gfp_mask|__GFP_ZERO); + new_node = genradix_alloc_node(gfp_mask); if (!new_node) return NULL; } @@ -133,7 +153,7 @@ void *__genradix_ptr_alloc(struct __genradix *radix, size_t offset, } if (new_node) - free_page((unsigned long) new_node); + genradix_free_node(new_node); return &n->data[offset]; } @@ -191,7 +211,7 @@ static void genradix_free_recurse(struct genradix_node *n, unsigned level) genradix_free_recurse(n->children[i], level - 1); } - free_page((unsigned long) n); + genradix_free_node(n); } int __genradix_prealloc(struct __genradix *radix, size_t size, diff --git a/lib/hexdump.c b/lib/hexdump.c index b1d55b669ae2..147133f8eb2f 100644 --- a/lib/hexdump.c +++ b/lib/hexdump.c @@ -270,25 +270,4 @@ void print_hex_dump(const char *level, const char *prefix_str, int prefix_type, } EXPORT_SYMBOL(print_hex_dump); -#if !defined(CONFIG_DYNAMIC_DEBUG) -/** - * print_hex_dump_bytes - shorthand form of print_hex_dump() with default params - * @prefix_str: string to prefix each line with; - * caller supplies trailing spaces for alignment if desired - * @prefix_type: controls whether prefix of an offset, address, or none - * is printed (%DUMP_PREFIX_OFFSET, %DUMP_PREFIX_ADDRESS, %DUMP_PREFIX_NONE) - * @buf: data blob to dump - * @len: number of bytes in the @buf - * - * Calls print_hex_dump(), with log level of KERN_DEBUG, - * rowsize of 16, groupsize of 1, and ASCII output included. - */ -void print_hex_dump_bytes(const char *prefix_str, int prefix_type, - const void *buf, size_t len) -{ - print_hex_dump(KERN_DEBUG, prefix_str, prefix_type, 16, 1, - buf, len, true); -} -EXPORT_SYMBOL(print_hex_dump_bytes); -#endif /* !defined(CONFIG_DYNAMIC_DEBUG) */ #endif /* defined(CONFIG_PRINTK) */ diff --git a/lib/ioremap.c b/lib/ioremap.c index 063213685563..0a2ffadc6d71 100644 --- a/lib/ioremap.c +++ b/lib/ioremap.c @@ -30,6 +30,8 @@ early_param("nohugeiomap", set_nohugeiomap); void __init ioremap_huge_init(void) { if (!ioremap_huge_disabled) { + if (arch_ioremap_p4d_supported()) + ioremap_p4d_capable = 1; if (arch_ioremap_pud_supported()) ioremap_pud_capable = 1; if (arch_ioremap_pmd_supported()) @@ -86,6 +88,9 @@ static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr, if ((end - addr) != PMD_SIZE) return 0; + if (!IS_ALIGNED(addr, PMD_SIZE)) + return 0; + if (!IS_ALIGNED(phys_addr, PMD_SIZE)) return 0; @@ -126,6 +131,9 @@ static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr, if ((end - addr) != PUD_SIZE) return 0; + if (!IS_ALIGNED(addr, PUD_SIZE)) + return 0; + if (!IS_ALIGNED(phys_addr, PUD_SIZE)) return 0; @@ -166,6 +174,9 @@ static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr, if ((end - addr) != P4D_SIZE) return 0; + if (!IS_ALIGNED(addr, P4D_SIZE)) + return 0; + if (!IS_ALIGNED(phys_addr, P4D_SIZE)) return 0; diff --git a/lib/iov_iter.c b/lib/iov_iter.c index f99c41d4eb54..639d5e7014c1 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -878,7 +878,7 @@ static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) head = compound_head(page); v += (page - head) << PAGE_SHIFT; - if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head)))) + if (likely(n <= v && v <= (page_size(head)))) return true; WARN_ON(1); return false; @@ -1634,9 +1634,9 @@ EXPORT_SYMBOL(dup_iter); * on-stack array was used or not (and regardless of whether this function * returns an error or not). * - * Return: 0 on success or negative error code on error. + * Return: Negative error code on error, bytes imported on success */ -int import_iovec(int type, const struct iovec __user * uvector, +ssize_t import_iovec(int type, const struct iovec __user * uvector, unsigned nr_segs, unsigned fast_segs, struct iovec **iov, struct iov_iter *i) { @@ -1652,16 +1652,17 @@ int import_iovec(int type, const struct iovec __user * uvector, } iov_iter_init(i, type, p, nr_segs, n); *iov = p == *iov ? NULL : p; - return 0; + return n; } EXPORT_SYMBOL(import_iovec); #ifdef CONFIG_COMPAT #include <linux/compat.h> -int compat_import_iovec(int type, const struct compat_iovec __user * uvector, - unsigned nr_segs, unsigned fast_segs, - struct iovec **iov, struct iov_iter *i) +ssize_t compat_import_iovec(int type, + const struct compat_iovec __user * uvector, + unsigned nr_segs, unsigned fast_segs, + struct iovec **iov, struct iov_iter *i) { ssize_t n; struct iovec *p; @@ -1675,7 +1676,7 @@ int compat_import_iovec(int type, const struct compat_iovec __user * uvector, } iov_iter_init(i, type, p, nr_segs, n); *iov = p == *iov ? NULL : p; - return 0; + return n; } #endif diff --git a/lib/jedec_ddr_data.c b/lib/jedec_ddr_data.c deleted file mode 100644 index d0b312e28d36..000000000000 --- a/lib/jedec_ddr_data.c +++ /dev/null @@ -1,132 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * DDR addressing details and AC timing parameters from JEDEC specs - * - * Copyright (C) 2012 Texas Instruments, Inc. - * - * Aneesh V <aneesh@ti.com> - */ - -#include <memory/jedec_ddr.h> -#include <linux/module.h> - -/* LPDDR2 addressing details from JESD209-2 section 2.4 */ -const struct lpddr2_addressing - lpddr2_jedec_addressing_table[NUM_DDR_ADDR_TABLE_ENTRIES] = { - {B4, T_REFI_15_6, T_RFC_90}, /* 64M */ - {B4, T_REFI_15_6, T_RFC_90}, /* 128M */ - {B4, T_REFI_7_8, T_RFC_90}, /* 256M */ - {B4, T_REFI_7_8, T_RFC_90}, /* 512M */ - {B8, T_REFI_7_8, T_RFC_130}, /* 1GS4 */ - {B8, T_REFI_3_9, T_RFC_130}, /* 2GS4 */ - {B8, T_REFI_3_9, T_RFC_130}, /* 4G */ - {B8, T_REFI_3_9, T_RFC_210}, /* 8G */ - {B4, T_REFI_7_8, T_RFC_130}, /* 1GS2 */ - {B4, T_REFI_3_9, T_RFC_130}, /* 2GS2 */ -}; -EXPORT_SYMBOL_GPL(lpddr2_jedec_addressing_table); - -/* LPDDR2 AC timing parameters from JESD209-2 section 12 */ -const struct lpddr2_timings - lpddr2_jedec_timings[NUM_DDR_TIMING_TABLE_ENTRIES] = { - /* Speed bin 400(200 MHz) */ - [0] = { - .max_freq = 200000000, - .min_freq = 10000000, - .tRPab = 21000, - .tRCD = 18000, - .tWR = 15000, - .tRAS_min = 42000, - .tRRD = 10000, - .tWTR = 10000, - .tXP = 7500, - .tRTP = 7500, - .tCKESR = 15000, - .tDQSCK_max = 5500, - .tFAW = 50000, - .tZQCS = 90000, - .tZQCL = 360000, - .tZQinit = 1000000, - .tRAS_max_ns = 70000, - .tDQSCK_max_derated = 6000, - }, - /* Speed bin 533(266 MHz) */ - [1] = { - .max_freq = 266666666, - .min_freq = 10000000, - .tRPab = 21000, - .tRCD = 18000, - .tWR = 15000, - .tRAS_min = 42000, - .tRRD = 10000, - .tWTR = 7500, - .tXP = 7500, - .tRTP = 7500, - .tCKESR = 15000, - .tDQSCK_max = 5500, - .tFAW = 50000, - .tZQCS = 90000, - .tZQCL = 360000, - .tZQinit = 1000000, - .tRAS_max_ns = 70000, - .tDQSCK_max_derated = 6000, - }, - /* Speed bin 800(400 MHz) */ - [2] = { - .max_freq = 400000000, - .min_freq = 10000000, - .tRPab = 21000, - .tRCD = 18000, - .tWR = 15000, - .tRAS_min = 42000, - .tRRD = 10000, - .tWTR = 7500, - .tXP = 7500, - .tRTP = 7500, - .tCKESR = 15000, - .tDQSCK_max = 5500, - .tFAW = 50000, - .tZQCS = 90000, - .tZQCL = 360000, - .tZQinit = 1000000, - .tRAS_max_ns = 70000, - .tDQSCK_max_derated = 6000, - }, - /* Speed bin 1066(533 MHz) */ - [3] = { - .max_freq = 533333333, - .min_freq = 10000000, - .tRPab = 21000, - .tRCD = 18000, - .tWR = 15000, - .tRAS_min = 42000, - .tRRD = 10000, - .tWTR = 7500, - .tXP = 7500, - .tRTP = 7500, - .tCKESR = 15000, - .tDQSCK_max = 5500, - .tFAW = 50000, - .tZQCS = 90000, - .tZQCL = 360000, - .tZQinit = 1000000, - .tRAS_max_ns = 70000, - .tDQSCK_max_derated = 5620, - }, -}; -EXPORT_SYMBOL_GPL(lpddr2_jedec_timings); - -const struct lpddr2_min_tck lpddr2_jedec_min_tck = { - .tRPab = 3, - .tRCD = 3, - .tWR = 3, - .tRASmin = 3, - .tRRD = 2, - .tWTR = 2, - .tXP = 2, - .tRTP = 2, - .tCKE = 3, - .tCKESR = 3, - .tFAW = 8 -}; -EXPORT_SYMBOL_GPL(lpddr2_jedec_min_tck); diff --git a/lib/kfifo.c b/lib/kfifo.c index 117ad0e7fbf4..70dab9ac7827 100644 --- a/lib/kfifo.c +++ b/lib/kfifo.c @@ -68,7 +68,8 @@ int __kfifo_init(struct __kfifo *fifo, void *buffer, { size /= esize; - size = roundup_pow_of_two(size); + if (!is_power_of_2(size)) + size = rounddown_pow_of_two(size); fifo->in = 0; fifo->out = 0; diff --git a/lib/kobject.c b/lib/kobject.c index f2ccdbac8ed9..83198cb37d8d 100644 --- a/lib/kobject.c +++ b/lib/kobject.c @@ -498,8 +498,10 @@ int kobject_rename(struct kobject *kobj, const char *new_name) kobj = kobject_get(kobj); if (!kobj) return -EINVAL; - if (!kobj->parent) + if (!kobj->parent) { + kobject_put(kobj); return -EINVAL; + } devpath = kobject_get_path(kobj, GFP_KERNEL); if (!devpath) { diff --git a/lib/list_sort.c b/lib/list_sort.c index 712ed1f4eb64..52f0c258c895 100644 --- a/lib/list_sort.c +++ b/lib/list_sort.c @@ -157,9 +157,11 @@ static void merge_final(void *priv, cmp_func cmp, struct list_head *head, * * The number of pending lists of size 2^k is determined by the * state of bit k of "count" plus two extra pieces of information: + * * - The state of bit k-1 (when k == 0, consider bit -1 always set), and * - Whether the higher-order bits are zero or non-zero (i.e. * is count >= 2^(k+1)). + * * There are six states we distinguish. "x" represents some arbitrary * bits, and "y" represents some arbitrary non-zero bits: * 0: 00x: 0 pending of size 2^k; x pending of sizes < 2^k diff --git a/lib/logic_pio.c b/lib/logic_pio.c index feea48fd1a0d..905027574e5d 100644 --- a/lib/logic_pio.c +++ b/lib/logic_pio.c @@ -35,7 +35,7 @@ int logic_pio_register_range(struct logic_pio_hwaddr *new_range) struct logic_pio_hwaddr *range; resource_size_t start; resource_size_t end; - resource_size_t mmio_sz = 0; + resource_size_t mmio_end = 0; resource_size_t iio_sz = MMIO_UPPER_LIMIT; int ret = 0; @@ -46,7 +46,7 @@ int logic_pio_register_range(struct logic_pio_hwaddr *new_range) end = new_range->hw_start + new_range->size; mutex_lock(&io_range_mutex); - list_for_each_entry_rcu(range, &io_range_list, list) { + list_for_each_entry(range, &io_range_list, list) { if (range->fwnode == new_range->fwnode) { /* range already there */ goto end_register; @@ -56,7 +56,7 @@ int logic_pio_register_range(struct logic_pio_hwaddr *new_range) /* for MMIO ranges we need to check for overlap */ if (start >= range->hw_start + range->size || end < range->hw_start) { - mmio_sz += range->size; + mmio_end = range->io_start + range->size; } else { ret = -EFAULT; goto end_register; @@ -69,16 +69,16 @@ int logic_pio_register_range(struct logic_pio_hwaddr *new_range) /* range not registered yet, check for available space */ if (new_range->flags == LOGIC_PIO_CPU_MMIO) { - if (mmio_sz + new_range->size - 1 > MMIO_UPPER_LIMIT) { + if (mmio_end + new_range->size - 1 > MMIO_UPPER_LIMIT) { /* if it's too big check if 64K space can be reserved */ - if (mmio_sz + SZ_64K - 1 > MMIO_UPPER_LIMIT) { + if (mmio_end + SZ_64K - 1 > MMIO_UPPER_LIMIT) { ret = -E2BIG; goto end_register; } new_range->size = SZ_64K; pr_warn("Requested IO range too big, new size set to 64K\n"); } - new_range->io_start = mmio_sz; + new_range->io_start = mmio_end; } else if (new_range->flags == LOGIC_PIO_INDIRECT) { if (iio_sz + new_range->size - 1 > IO_SPACE_LIMIT) { ret = -E2BIG; @@ -99,6 +99,20 @@ end_register: } /** + * logic_pio_unregister_range - unregister a logical PIO range for a host + * @range: pointer to the IO range which has been already registered. + * + * Unregister a previously-registered IO range node. + */ +void logic_pio_unregister_range(struct logic_pio_hwaddr *range) +{ + mutex_lock(&io_range_mutex); + list_del_rcu(&range->list); + mutex_unlock(&io_range_mutex); + synchronize_rcu(); +} + +/** * find_io_range_by_fwnode - find logical PIO range for given FW node * @fwnode: FW node handle associated with logical PIO range * @@ -108,26 +122,38 @@ end_register: */ struct logic_pio_hwaddr *find_io_range_by_fwnode(struct fwnode_handle *fwnode) { - struct logic_pio_hwaddr *range; + struct logic_pio_hwaddr *range, *found_range = NULL; + rcu_read_lock(); list_for_each_entry_rcu(range, &io_range_list, list) { - if (range->fwnode == fwnode) - return range; + if (range->fwnode == fwnode) { + found_range = range; + break; + } } - return NULL; + rcu_read_unlock(); + + return found_range; } /* Return a registered range given an input PIO token */ static struct logic_pio_hwaddr *find_io_range(unsigned long pio) { - struct logic_pio_hwaddr *range; + struct logic_pio_hwaddr *range, *found_range = NULL; + rcu_read_lock(); list_for_each_entry_rcu(range, &io_range_list, list) { - if (in_range(pio, range->io_start, range->size)) - return range; + if (in_range(pio, range->io_start, range->size)) { + found_range = range; + break; + } } - pr_err("PIO entry token %lx invalid\n", pio); - return NULL; + rcu_read_unlock(); + + if (!found_range) + pr_err("PIO entry token 0x%lx invalid\n", pio); + + return found_range; } /** @@ -180,14 +206,23 @@ unsigned long logic_pio_trans_cpuaddr(resource_size_t addr) { struct logic_pio_hwaddr *range; + rcu_read_lock(); list_for_each_entry_rcu(range, &io_range_list, list) { if (range->flags != LOGIC_PIO_CPU_MMIO) continue; - if (in_range(addr, range->hw_start, range->size)) - return addr - range->hw_start + range->io_start; + if (in_range(addr, range->hw_start, range->size)) { + unsigned long cpuaddr; + + cpuaddr = addr - range->hw_start + range->io_start; + + rcu_read_unlock(); + return cpuaddr; + } } - pr_err("addr %llx not registered in io_range_list\n", - (unsigned long long) addr); + rcu_read_unlock(); + + pr_err("addr %pa not registered in io_range_list\n", &addr); + return ~0UL; } diff --git a/lib/lz4/lz4hc_compress.c b/lib/lz4/lz4hc_compress.c index 176f03b83e56..1b61d874e337 100644 --- a/lib/lz4/lz4hc_compress.c +++ b/lib/lz4/lz4hc_compress.c @@ -663,7 +663,6 @@ static void LZ4HC_setExternalDict( /* match referencing will resume from there */ ctxPtr->nextToUpdate = ctxPtr->dictLimit; } -EXPORT_SYMBOL(LZ4HC_setExternalDict); static int LZ4_compressHC_continue_generic( LZ4_streamHC_t *LZ4_streamHCPtr, diff --git a/lib/lzo/lzo1x_compress.c b/lib/lzo/lzo1x_compress.c index ba16c08e8cb9..717c940112f9 100644 --- a/lib/lzo/lzo1x_compress.c +++ b/lib/lzo/lzo1x_compress.c @@ -83,17 +83,19 @@ next: ALIGN((uintptr_t)ir, 4)) && (ir < limit) && (*ir == 0)) ir++; - for (; (ir + 4) <= limit; ir += 4) { - dv = *((u32 *)ir); - if (dv) { + if (IS_ALIGNED((uintptr_t)ir, 4)) { + for (; (ir + 4) <= limit; ir += 4) { + dv = *((u32 *)ir); + if (dv) { # if defined(__LITTLE_ENDIAN) - ir += __builtin_ctz(dv) >> 3; + ir += __builtin_ctz(dv) >> 3; # elif defined(__BIG_ENDIAN) - ir += __builtin_clz(dv) >> 3; + ir += __builtin_clz(dv) >> 3; # else # error "missing endian definition" # endif - break; + break; + } } } #endif diff --git a/lib/mpi/longlong.h b/lib/mpi/longlong.h index 08c60d10747f..2dceaca27489 100644 --- a/lib/mpi/longlong.h +++ b/lib/mpi/longlong.h @@ -397,8 +397,8 @@ do { \ #define add_ssaaaa(sh, sl, ah, al, bh, bl) \ __asm__ ("addl %5,%1\n" \ "adcl %3,%0" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "%0" ((USItype)(ah)), \ "g" ((USItype)(bh)), \ "%1" ((USItype)(al)), \ @@ -406,22 +406,22 @@ do { \ #define sub_ddmmss(sh, sl, ah, al, bh, bl) \ __asm__ ("subl %5,%1\n" \ "sbbl %3,%0" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "0" ((USItype)(ah)), \ "g" ((USItype)(bh)), \ "1" ((USItype)(al)), \ "g" ((USItype)(bl))) #define umul_ppmm(w1, w0, u, v) \ __asm__ ("mull %3" \ - : "=a" ((USItype)(w0)), \ - "=d" ((USItype)(w1)) \ + : "=a" (w0), \ + "=d" (w1) \ : "%0" ((USItype)(u)), \ "rm" ((USItype)(v))) #define udiv_qrnnd(q, r, n1, n0, d) \ __asm__ ("divl %4" \ - : "=a" ((USItype)(q)), \ - "=d" ((USItype)(r)) \ + : "=a" (q), \ + "=d" (r) \ : "0" ((USItype)(n0)), \ "1" ((USItype)(n1)), \ "rm" ((USItype)(d))) @@ -639,30 +639,12 @@ do { \ ************** MIPS ***************** ***************************************/ #if defined(__mips__) && W_TYPE_SIZE == 32 -#if (__GNUC__ >= 5) || (__GNUC__ >= 4 && __GNUC_MINOR__ >= 4) #define umul_ppmm(w1, w0, u, v) \ do { \ UDItype __ll = (UDItype)(u) * (v); \ w1 = __ll >> 32; \ w0 = __ll; \ } while (0) -#elif __GNUC__ > 2 || __GNUC_MINOR__ >= 7 -#define umul_ppmm(w1, w0, u, v) \ - __asm__ ("multu %2,%3" \ - : "=l" ((USItype)(w0)), \ - "=h" ((USItype)(w1)) \ - : "d" ((USItype)(u)), \ - "d" ((USItype)(v))) -#else -#define umul_ppmm(w1, w0, u, v) \ - __asm__ ("multu %2,%3\n" \ - "mflo %0\n" \ - "mfhi %1" \ - : "=d" ((USItype)(w0)), \ - "=d" ((USItype)(w1)) \ - : "d" ((USItype)(u)), \ - "d" ((USItype)(v))) -#endif #define UMUL_TIME 10 #define UDIV_TIME 100 #endif /* __mips__ */ @@ -687,7 +669,7 @@ do { \ : "d" ((UDItype)(u)), \ "d" ((UDItype)(v))); \ } while (0) -#elif (__GNUC__ >= 5) || (__GNUC__ >= 4 && __GNUC_MINOR__ >= 4) +#else #define umul_ppmm(w1, w0, u, v) \ do { \ typedef unsigned int __ll_UTItype __attribute__((mode(TI))); \ @@ -695,22 +677,6 @@ do { \ w1 = __ll >> 64; \ w0 = __ll; \ } while (0) -#elif __GNUC__ > 2 || __GNUC_MINOR__ >= 7 -#define umul_ppmm(w1, w0, u, v) \ - __asm__ ("dmultu %2,%3" \ - : "=l" ((UDItype)(w0)), \ - "=h" ((UDItype)(w1)) \ - : "d" ((UDItype)(u)), \ - "d" ((UDItype)(v))) -#else -#define umul_ppmm(w1, w0, u, v) \ - __asm__ ("dmultu %2,%3\n" \ - "mflo %0\n" \ - "mfhi %1" \ - : "=d" ((UDItype)(w0)), \ - "=d" ((UDItype)(w1)) \ - : "d" ((UDItype)(u)), \ - "d" ((UDItype)(v))) #endif #define UMUL_TIME 20 #define UDIV_TIME 140 diff --git a/lib/mpi/mpi-pow.c b/lib/mpi/mpi-pow.c index 82b19e4f1189..2fd7a46d55ec 100644 --- a/lib/mpi/mpi-pow.c +++ b/lib/mpi/mpi-pow.c @@ -24,6 +24,7 @@ int mpi_powm(MPI res, MPI base, MPI exp, MPI mod) { mpi_ptr_t mp_marker = NULL, bp_marker = NULL, ep_marker = NULL; + struct karatsuba_ctx karactx = {}; mpi_ptr_t xp_marker = NULL; mpi_ptr_t tspace = NULL; mpi_ptr_t rp, ep, mp, bp; @@ -150,13 +151,11 @@ int mpi_powm(MPI res, MPI base, MPI exp, MPI mod) int c; mpi_limb_t e; mpi_limb_t carry_limb; - struct karatsuba_ctx karactx; xp = xp_marker = mpi_alloc_limb_space(2 * (msize + 1)); if (!xp) goto enomem; - memset(&karactx, 0, sizeof karactx); negative_result = (ep[0] & 1) && base->sign; i = esize - 1; @@ -281,8 +280,6 @@ int mpi_powm(MPI res, MPI base, MPI exp, MPI mod) if (mod_shift_cnt) mpihelp_rshift(rp, rp, rsize, mod_shift_cnt); MPN_NORMALIZE(rp, rsize); - - mpihelp_release_karatsuba_ctx(&karactx); } if (negative_result && rsize) { @@ -299,6 +296,7 @@ int mpi_powm(MPI res, MPI base, MPI exp, MPI mod) leave: rc = 0; enomem: + mpihelp_release_karatsuba_ctx(&karactx); if (assign_rp) mpi_assign_limb_space(res, rp, size); if (mp_marker) diff --git a/lib/notifier-error-inject.c b/lib/notifier-error-inject.c index 3d2ba7cf83f4..21016b32d313 100644 --- a/lib/notifier-error-inject.c +++ b/lib/notifier-error-inject.c @@ -59,33 +59,22 @@ struct dentry *notifier_err_inject_init(const char *name, struct dentry *parent, err_inject->nb.priority = priority; dir = debugfs_create_dir(name, parent); - if (!dir) - return ERR_PTR(-ENOMEM); actions_dir = debugfs_create_dir("actions", dir); - if (!actions_dir) - goto fail; for (action = err_inject->actions; action->name; action++) { struct dentry *action_dir; action_dir = debugfs_create_dir(action->name, actions_dir); - if (!action_dir) - goto fail; /* * Create debugfs r/w file containing action->error. If * notifier call chain is called with action->val, it will * fail with the error code */ - if (!debugfs_create_errno("error", mode, action_dir, - &action->error)) - goto fail; + debugfs_create_errno("error", mode, action_dir, &action->error); } return dir; -fail: - debugfs_remove_recursive(dir); - return ERR_PTR(-ENOMEM); } EXPORT_SYMBOL_GPL(notifier_err_inject_init); diff --git a/lib/objagg.c b/lib/objagg.c index 576be22e86de..55621fb82e0a 100644 --- a/lib/objagg.c +++ b/lib/objagg.c @@ -605,12 +605,10 @@ const struct objagg_stats *objagg_stats_get(struct objagg *objagg) { struct objagg_stats *objagg_stats; struct objagg_obj *objagg_obj; - size_t alloc_size; int i; - alloc_size = sizeof(*objagg_stats) + - sizeof(objagg_stats->stats_info[0]) * objagg->obj_count; - objagg_stats = kzalloc(alloc_size, GFP_KERNEL); + objagg_stats = kzalloc(struct_size(objagg_stats, stats_info, + objagg->obj_count), GFP_KERNEL); if (!objagg_stats) return ERR_PTR(-ENOMEM); diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 071a76c7bac0..4f6c6ebbbbde 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -70,11 +70,14 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release, return -ENOMEM; ref->force_atomic = flags & PERCPU_REF_INIT_ATOMIC; + ref->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT; - if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) + if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) { ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC; - else + ref->allow_reinit = true; + } else { start_count += PERCPU_COUNT_BIAS; + } if (flags & PERCPU_REF_INIT_DEAD) ref->percpu_count_ptr |= __PERCPU_REF_DEAD; @@ -120,6 +123,9 @@ static void percpu_ref_call_confirm_rcu(struct rcu_head *rcu) ref->confirm_switch = NULL; wake_up_all(&percpu_ref_switch_waitq); + if (!ref->allow_reinit) + percpu_ref_exit(ref); + /* drop ref from percpu_ref_switch_to_atomic() */ percpu_ref_put(ref); } @@ -195,6 +201,9 @@ static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref) if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC)) return; + if (WARN_ON_ONCE(!ref->allow_reinit)) + return; + atomic_long_add(PERCPU_COUNT_BIAS, &ref->count); /* diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile index e723eacf7868..0083b5cc646c 100644 --- a/lib/raid6/Makefile +++ b/lib/raid6/Makefile @@ -12,9 +12,6 @@ raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o hostprogs-y += mktables -quiet_cmd_unroll = UNROLL $@ - cmd_unroll = $(AWK) -f$(srctree)/$(src)/unroll.awk -vN=$(UNROLL) < $< > $@ - ifeq ($(CONFIG_ALTIVEC),y) altivec_flags := -maltivec $(call cc-option,-mabi=altivec) @@ -26,7 +23,6 @@ CFLAGS_REMOVE_altivec1.o += -msoft-float CFLAGS_REMOVE_altivec2.o += -msoft-float CFLAGS_REMOVE_altivec4.o += -msoft-float CFLAGS_REMOVE_altivec8.o += -msoft-float -CFLAGS_REMOVE_altivec8.o += -msoft-float CFLAGS_REMOVE_vpermxor1.o += -msoft-float CFLAGS_REMOVE_vpermxor2.o += -msoft-float CFLAGS_REMOVE_vpermxor4.o += -msoft-float @@ -51,111 +47,39 @@ CFLAGS_REMOVE_neon8.o += -mgeneral-regs-only endif endif -targets += int1.c -$(obj)/int1.c: UNROLL := 1 -$(obj)/int1.c: $(src)/int.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -targets += int2.c -$(obj)/int2.c: UNROLL := 2 -$(obj)/int2.c: $(src)/int.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -targets += int4.c -$(obj)/int4.c: UNROLL := 4 -$(obj)/int4.c: $(src)/int.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -targets += int8.c -$(obj)/int8.c: UNROLL := 8 -$(obj)/int8.c: $(src)/int.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -targets += int16.c -$(obj)/int16.c: UNROLL := 16 -$(obj)/int16.c: $(src)/int.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) +quiet_cmd_unroll = UNROLL $@ + cmd_unroll = $(AWK) -f$(srctree)/$(src)/unroll.awk -vN=$* < $< > $@ -targets += int32.c -$(obj)/int32.c: UNROLL := 32 -$(obj)/int32.c: $(src)/int.uc $(src)/unroll.awk FORCE +targets += int1.c int2.c int4.c int8.c int16.c int32.c +$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE $(call if_changed,unroll) CFLAGS_altivec1.o += $(altivec_flags) -targets += altivec1.c -$(obj)/altivec1.c: UNROLL := 1 -$(obj)/altivec1.c: $(src)/altivec.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - CFLAGS_altivec2.o += $(altivec_flags) -targets += altivec2.c -$(obj)/altivec2.c: UNROLL := 2 -$(obj)/altivec2.c: $(src)/altivec.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - CFLAGS_altivec4.o += $(altivec_flags) -targets += altivec4.c -$(obj)/altivec4.c: UNROLL := 4 -$(obj)/altivec4.c: $(src)/altivec.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - CFLAGS_altivec8.o += $(altivec_flags) -targets += altivec8.c -$(obj)/altivec8.c: UNROLL := 8 -$(obj)/altivec8.c: $(src)/altivec.uc $(src)/unroll.awk FORCE +targets += altivec1.c altivec2.c altivec4.c altivec8.c +$(obj)/altivec%.c: $(src)/altivec.uc $(src)/unroll.awk FORCE $(call if_changed,unroll) CFLAGS_vpermxor1.o += $(altivec_flags) -targets += vpermxor1.c -$(obj)/vpermxor1.c: UNROLL := 1 -$(obj)/vpermxor1.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - CFLAGS_vpermxor2.o += $(altivec_flags) -targets += vpermxor2.c -$(obj)/vpermxor2.c: UNROLL := 2 -$(obj)/vpermxor2.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - CFLAGS_vpermxor4.o += $(altivec_flags) -targets += vpermxor4.c -$(obj)/vpermxor4.c: UNROLL := 4 -$(obj)/vpermxor4.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - CFLAGS_vpermxor8.o += $(altivec_flags) -targets += vpermxor8.c -$(obj)/vpermxor8.c: UNROLL := 8 -$(obj)/vpermxor8.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE +targets += vpermxor1.c vpermxor2.c vpermxor4.c vpermxor8.c +$(obj)/vpermxor%.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE $(call if_changed,unroll) CFLAGS_neon1.o += $(NEON_FLAGS) -targets += neon1.c -$(obj)/neon1.c: UNROLL := 1 -$(obj)/neon1.c: $(src)/neon.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - CFLAGS_neon2.o += $(NEON_FLAGS) -targets += neon2.c -$(obj)/neon2.c: UNROLL := 2 -$(obj)/neon2.c: $(src)/neon.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - CFLAGS_neon4.o += $(NEON_FLAGS) -targets += neon4.c -$(obj)/neon4.c: UNROLL := 4 -$(obj)/neon4.c: $(src)/neon.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - CFLAGS_neon8.o += $(NEON_FLAGS) -targets += neon8.c -$(obj)/neon8.c: UNROLL := 8 -$(obj)/neon8.c: $(src)/neon.uc $(src)/unroll.awk FORCE +targets += neon1.c neon2.c neon4.c neon8.c +$(obj)/neon%.c: $(src)/neon.uc $(src)/unroll.awk FORCE $(call if_changed,unroll) targets += s390vx8.c -$(obj)/s390vx8.c: UNROLL := 8 -$(obj)/s390vx8.c: $(src)/s390vx.uc $(src)/unroll.awk FORCE +$(obj)/s390vx%.c: $(src)/s390vx.uc $(src)/unroll.awk FORCE $(call if_changed,unroll) quiet_cmd_mktable = TABLE $@ diff --git a/lib/raid6/s390vx.uc b/lib/raid6/s390vx.uc index 914ebe98fc21..9e597e1f91a4 100644 --- a/lib/raid6/s390vx.uc +++ b/lib/raid6/s390vx.uc @@ -60,7 +60,7 @@ static inline void LOAD_DATA(int x, u8 *ptr) typedef struct { u8 _[16 * $#]; } addrtype; register addrtype *__ptr asm("1") = (addrtype *) ptr; - asm volatile ("VLM %2,%3,0,%r1" + asm volatile ("VLM %2,%3,0,%1" : : "m" (*__ptr), "a" (__ptr), "i" (x), "i" (x + $# - 1)); } diff --git a/lib/rbtree.c b/lib/rbtree.c index 1ef6e25d031c..abc86c6a3177 100644 --- a/lib/rbtree.c +++ b/lib/rbtree.c @@ -83,14 +83,10 @@ __rb_rotate_set_parents(struct rb_node *old, struct rb_node *new, static __always_inline void __rb_insert(struct rb_node *node, struct rb_root *root, - bool newleft, struct rb_node **leftmost, void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) { struct rb_node *parent = rb_red_parent(node), *gparent, *tmp; - if (newleft) - *leftmost = node; - while (true) { /* * Loop invariant: node is red. @@ -437,38 +433,19 @@ static const struct rb_augment_callbacks dummy_callbacks = { void rb_insert_color(struct rb_node *node, struct rb_root *root) { - __rb_insert(node, root, false, NULL, dummy_rotate); + __rb_insert(node, root, dummy_rotate); } EXPORT_SYMBOL(rb_insert_color); void rb_erase(struct rb_node *node, struct rb_root *root) { struct rb_node *rebalance; - rebalance = __rb_erase_augmented(node, root, - NULL, &dummy_callbacks); + rebalance = __rb_erase_augmented(node, root, &dummy_callbacks); if (rebalance) ____rb_erase_color(rebalance, root, dummy_rotate); } EXPORT_SYMBOL(rb_erase); -void rb_insert_color_cached(struct rb_node *node, - struct rb_root_cached *root, bool leftmost) -{ - __rb_insert(node, &root->rb_root, leftmost, - &root->rb_leftmost, dummy_rotate); -} -EXPORT_SYMBOL(rb_insert_color_cached); - -void rb_erase_cached(struct rb_node *node, struct rb_root_cached *root) -{ - struct rb_node *rebalance; - rebalance = __rb_erase_augmented(node, &root->rb_root, - &root->rb_leftmost, &dummy_callbacks); - if (rebalance) - ____rb_erase_color(rebalance, &root->rb_root, dummy_rotate); -} -EXPORT_SYMBOL(rb_erase_cached); - /* * Augmented rbtree manipulation functions. * @@ -477,10 +454,9 @@ EXPORT_SYMBOL(rb_erase_cached); */ void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, - bool newleft, struct rb_node **leftmost, void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) { - __rb_insert(node, root, newleft, leftmost, augment_rotate); + __rb_insert(node, root, augment_rotate); } EXPORT_SYMBOL(__rb_insert_augmented); @@ -591,16 +567,6 @@ void rb_replace_node(struct rb_node *victim, struct rb_node *new, } EXPORT_SYMBOL(rb_replace_node); -void rb_replace_node_cached(struct rb_node *victim, struct rb_node *new, - struct rb_root_cached *root) -{ - rb_replace_node(victim, new, &root->rb_root); - - if (root->rb_leftmost == victim) - root->rb_leftmost = new; -} -EXPORT_SYMBOL(rb_replace_node_cached); - void rb_replace_node_rcu(struct rb_node *victim, struct rb_node *new, struct rb_root *root) { diff --git a/lib/rbtree_test.c b/lib/rbtree_test.c index 62b8ee92643d..41ae3c7570d3 100644 --- a/lib/rbtree_test.c +++ b/lib/rbtree_test.c @@ -77,26 +77,10 @@ static inline void erase_cached(struct test_node *node, struct rb_root_cached *r } -static inline u32 augment_recompute(struct test_node *node) -{ - u32 max = node->val, child_augmented; - if (node->rb.rb_left) { - child_augmented = rb_entry(node->rb.rb_left, struct test_node, - rb)->augmented; - if (max < child_augmented) - max = child_augmented; - } - if (node->rb.rb_right) { - child_augmented = rb_entry(node->rb.rb_right, struct test_node, - rb)->augmented; - if (max < child_augmented) - max = child_augmented; - } - return max; -} +#define NODE_VAL(node) ((node)->val) -RB_DECLARE_CALLBACKS(static, augment_callbacks, struct test_node, rb, - u32, augmented, augment_recompute) +RB_DECLARE_CALLBACKS_MAX(static, augment_callbacks, + struct test_node, rb, u32, augmented, NODE_VAL) static void insert_augmented(struct test_node *node, struct rb_root_cached *root) @@ -238,7 +222,20 @@ static void check_augmented(int nr_nodes) check(nr_nodes); for (rb = rb_first(&root.rb_root); rb; rb = rb_next(rb)) { struct test_node *node = rb_entry(rb, struct test_node, rb); - WARN_ON_ONCE(node->augmented != augment_recompute(node)); + u32 subtree, max = node->val; + if (node->rb.rb_left) { + subtree = rb_entry(node->rb.rb_left, struct test_node, + rb)->augmented; + if (max < subtree) + max = subtree; + } + if (node->rb.rb_right) { + subtree = rb_entry(node->rb.rb_right, struct test_node, + rb)->augmented; + if (max < subtree) + max = subtree; + } + WARN_ON_ONCE(node->augmented != max); } } diff --git a/lib/reed_solomon/Makefile b/lib/reed_solomon/Makefile index ba9d7a3329eb..5d4fa68f26cb 100644 --- a/lib/reed_solomon/Makefile +++ b/lib/reed_solomon/Makefile @@ -4,4 +4,4 @@ # obj-$(CONFIG_REED_SOLOMON) += reed_solomon.o - +obj-$(CONFIG_REED_SOLOMON_TEST) += test_rslib.o diff --git a/lib/reed_solomon/decode_rs.c b/lib/reed_solomon/decode_rs.c index 1db74eb098d0..805de84ae83d 100644 --- a/lib/reed_solomon/decode_rs.c +++ b/lib/reed_solomon/decode_rs.c @@ -22,6 +22,7 @@ uint16_t *index_of = rs->index_of; uint16_t u, q, tmp, num1, num2, den, discr_r, syn_error; int count = 0; + int num_corrected; uint16_t msk = (uint16_t) rs->nn; /* @@ -39,11 +40,21 @@ /* Check length parameter for validity */ pad = nn - nroots - len; - BUG_ON(pad < 0 || pad >= nn); + BUG_ON(pad < 0 || pad >= nn - nroots); /* Does the caller provide the syndrome ? */ - if (s != NULL) - goto decode; + if (s != NULL) { + for (i = 0; i < nroots; i++) { + /* The syndrome is in index form, + * so nn represents zero + */ + if (s[i] != nn) + goto decode; + } + + /* syndrome is zero, no errors to correct */ + return 0; + } /* form the syndromes; i.e., evaluate data(x) at roots of * g(x) */ @@ -88,8 +99,7 @@ /* if syndrome is zero, data[] is a codeword and there are no * errors to correct. So return data[] unmodified */ - count = 0; - goto finish; + return 0; } decode: @@ -99,9 +109,9 @@ if (no_eras > 0) { /* Init lambda to be the erasure locator polynomial */ lambda[1] = alpha_to[rs_modnn(rs, - prim * (nn - 1 - eras_pos[0]))]; + prim * (nn - 1 - (eras_pos[0] + pad)))]; for (i = 1; i < no_eras; i++) { - u = rs_modnn(rs, prim * (nn - 1 - eras_pos[i])); + u = rs_modnn(rs, prim * (nn - 1 - (eras_pos[i] + pad))); for (j = i + 1; j > 0; j--) { tmp = index_of[lambda[j - 1]]; if (tmp != nn) { @@ -175,6 +185,15 @@ if (lambda[i] != nn) deg_lambda = i; } + + if (deg_lambda == 0) { + /* + * deg(lambda) is zero even though the syndrome is non-zero + * => uncorrectable error detected + */ + return -EBADMSG; + } + /* Find roots of error+erasure locator polynomial by Chien search */ memcpy(®[1], &lambda[1], nroots * sizeof(reg[0])); count = 0; /* Number of roots of lambda(x) */ @@ -188,6 +207,12 @@ } if (q != 0) continue; /* Not a root */ + + if (k < pad) { + /* Impossible error location. Uncorrectable error. */ + return -EBADMSG; + } + /* store root (index-form) and error location number */ root[count] = i; loc[count] = k; @@ -202,8 +227,7 @@ * deg(lambda) unequal to number of roots => uncorrectable * error detected */ - count = -EBADMSG; - goto finish; + return -EBADMSG; } /* * Compute err+eras evaluator poly omega(x) = s(x)*lambda(x) (modulo @@ -223,7 +247,9 @@ /* * Compute error values in poly-form. num1 = omega(inv(X(l))), num2 = * inv(X(l))**(fcr-1) and den = lambda_pr(inv(X(l))) all in poly-form + * Note: we reuse the buffer for b to store the correction pattern */ + num_corrected = 0; for (j = count - 1; j >= 0; j--) { num1 = 0; for (i = deg_omega; i >= 0; i--) { @@ -231,6 +257,13 @@ num1 ^= alpha_to[rs_modnn(rs, omega[i] + i * root[j])]; } + + if (num1 == 0) { + /* Nothing to correct at this position */ + b[j] = 0; + continue; + } + num2 = alpha_to[rs_modnn(rs, root[j] * (fcr - 1) + nn)]; den = 0; @@ -242,30 +275,52 @@ i * root[j])]; } } - /* Apply error to data */ - if (num1 != 0 && loc[j] >= pad) { - uint16_t cor = alpha_to[rs_modnn(rs,index_of[num1] + - index_of[num2] + - nn - index_of[den])]; - /* Store the error correction pattern, if a - * correction buffer is available */ - if (corr) { - corr[j] = cor; - } else { - /* If a data buffer is given and the - * error is inside the message, - * correct it */ - if (data && (loc[j] < (nn - nroots))) - data[loc[j] - pad] ^= cor; - } + + b[j] = alpha_to[rs_modnn(rs, index_of[num1] + + index_of[num2] + + nn - index_of[den])]; + num_corrected++; + } + + /* + * We compute the syndrome of the 'error' and check that it matches + * the syndrome of the received word + */ + for (i = 0; i < nroots; i++) { + tmp = 0; + for (j = 0; j < count; j++) { + if (b[j] == 0) + continue; + + k = (fcr + i) * prim * (nn-loc[j]-1); + tmp ^= alpha_to[rs_modnn(rs, index_of[b[j]] + k)]; } + + if (tmp != alpha_to[s[i]]) + return -EBADMSG; } -finish: - if (eras_pos != NULL) { - for (i = 0; i < count; i++) - eras_pos[i] = loc[i] - pad; + /* + * Store the error correction pattern, if a + * correction buffer is available + */ + if (corr && eras_pos) { + j = 0; + for (i = 0; i < count; i++) { + if (b[i]) { + corr[j] = b[i]; + eras_pos[j++] = loc[i] - pad; + } + } + } else if (data && par) { + /* Apply error to data and parity */ + for (i = 0; i < count; i++) { + if (loc[i] < (nn - nroots)) + data[loc[i] - pad] ^= b[i]; + else + par[loc[i] - pad - len] ^= b[i]; + } } - return count; + return num_corrected; } diff --git a/lib/reed_solomon/reed_solomon.c b/lib/reed_solomon/reed_solomon.c index e5fdc8b9e856..bbc01bad3053 100644 --- a/lib/reed_solomon/reed_solomon.c +++ b/lib/reed_solomon/reed_solomon.c @@ -340,7 +340,8 @@ EXPORT_SYMBOL_GPL(encode_rs8); * @data: data field of a given type * @par: received parity data field * @len: data length - * @s: syndrome data field (if NULL, syndrome is calculated) + * @s: syndrome data field, must be in index form + * (if NULL, syndrome is calculated) * @no_eras: number of erasures * @eras_pos: position of erasures, can be NULL * @invmsk: invert data mask (will be xored on data, not on parity!) @@ -354,7 +355,8 @@ EXPORT_SYMBOL_GPL(encode_rs8); * decoding, so the caller has to ensure that decoder invocations are * serialized. * - * Returns the number of corrected bits or -EBADMSG for uncorrectable errors. + * Returns the number of corrected symbols or -EBADMSG for uncorrectable + * errors. The count includes errors in the parity. */ int decode_rs8(struct rs_control *rsc, uint8_t *data, uint16_t *par, int len, uint16_t *s, int no_eras, int *eras_pos, uint16_t invmsk, @@ -391,7 +393,8 @@ EXPORT_SYMBOL_GPL(encode_rs16); * @data: data field of a given type * @par: received parity data field * @len: data length - * @s: syndrome data field (if NULL, syndrome is calculated) + * @s: syndrome data field, must be in index form + * (if NULL, syndrome is calculated) * @no_eras: number of erasures * @eras_pos: position of erasures, can be NULL * @invmsk: invert data mask (will be xored on data, not on parity!) @@ -403,7 +406,8 @@ EXPORT_SYMBOL_GPL(encode_rs16); * decoding, so the caller has to ensure that decoder invocations are * serialized. * - * Returns the number of corrected bits or -EBADMSG for uncorrectable errors. + * Returns the number of corrected symbols or -EBADMSG for uncorrectable + * errors. The count includes errors in the parity. */ int decode_rs16(struct rs_control *rsc, uint16_t *data, uint16_t *par, int len, uint16_t *s, int no_eras, int *eras_pos, uint16_t invmsk, diff --git a/lib/reed_solomon/test_rslib.c b/lib/reed_solomon/test_rslib.c new file mode 100644 index 000000000000..4eb29f365ece --- /dev/null +++ b/lib/reed_solomon/test_rslib.c @@ -0,0 +1,518 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Tests for Generic Reed Solomon encoder / decoder library + * + * Written by Ferdinand Blomqvist + * Based on previous work by Phil Karn, KA9Q + */ +#include <linux/rslib.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/random.h> +#include <linux/slab.h> + +enum verbosity { + V_SILENT, + V_PROGRESS, + V_CSUMMARY +}; + +enum method { + CORR_BUFFER, + CALLER_SYNDROME, + IN_PLACE +}; + +#define __param(type, name, init, msg) \ + static type name = init; \ + module_param(name, type, 0444); \ + MODULE_PARM_DESC(name, msg) + +__param(int, v, V_PROGRESS, "Verbosity level"); +__param(int, ewsc, 1, "Erasures without symbol corruption"); +__param(int, bc, 1, "Test for correct behaviour beyond error correction capacity"); + +struct etab { + int symsize; + int genpoly; + int fcs; + int prim; + int nroots; + int ntrials; +}; + +/* List of codes to test */ +static struct etab Tab[] = { + {2, 0x7, 1, 1, 1, 100000 }, + {3, 0xb, 1, 1, 2, 100000 }, + {3, 0xb, 1, 1, 3, 100000 }, + {3, 0xb, 2, 1, 4, 100000 }, + {4, 0x13, 1, 1, 4, 10000 }, + {5, 0x25, 1, 1, 6, 1000 }, + {6, 0x43, 3, 1, 8, 1000 }, + {7, 0x89, 1, 1, 14, 500 }, + {8, 0x11d, 1, 1, 30, 100 }, + {8, 0x187, 112, 11, 32, 100 }, + {9, 0x211, 1, 1, 33, 80 }, + {0, 0, 0, 0, 0, 0}, +}; + + +struct estat { + int dwrong; + int irv; + int wepos; + int nwords; +}; + +struct bcstat { + int rfail; + int rsuccess; + int noncw; + int nwords; +}; + +struct wspace { + uint16_t *c; /* sent codeword */ + uint16_t *r; /* received word */ + uint16_t *s; /* syndrome */ + uint16_t *corr; /* correction buffer */ + int *errlocs; + int *derrlocs; +}; + +struct pad { + int mult; + int shift; +}; + +static struct pad pad_coef[] = { + { 0, 0 }, + { 1, 2 }, + { 1, 1 }, + { 3, 2 }, + { 1, 0 }, +}; + +static void free_ws(struct wspace *ws) +{ + if (!ws) + return; + + kfree(ws->errlocs); + kfree(ws->c); + kfree(ws); +} + +static struct wspace *alloc_ws(struct rs_codec *rs) +{ + int nroots = rs->nroots; + struct wspace *ws; + int nn = rs->nn; + + ws = kzalloc(sizeof(*ws), GFP_KERNEL); + if (!ws) + return NULL; + + ws->c = kmalloc_array(2 * (nn + nroots), + sizeof(uint16_t), GFP_KERNEL); + if (!ws->c) + goto err; + + ws->r = ws->c + nn; + ws->s = ws->r + nn; + ws->corr = ws->s + nroots; + + ws->errlocs = kmalloc_array(nn + nroots, sizeof(int), GFP_KERNEL); + if (!ws->errlocs) + goto err; + + ws->derrlocs = ws->errlocs + nn; + return ws; + +err: + free_ws(ws); + return NULL; +} + + +/* + * Generates a random codeword and stores it in c. Generates random errors and + * erasures, and stores the random word with errors in r. Erasure positions are + * stored in derrlocs, while errlocs has one of three values in every position: + * + * 0 if there is no error in this position; + * 1 if there is a symbol error in this position; + * 2 if there is an erasure without symbol corruption. + * + * Returns the number of corrupted symbols. + */ +static int get_rcw_we(struct rs_control *rs, struct wspace *ws, + int len, int errs, int eras) +{ + int nroots = rs->codec->nroots; + int *derrlocs = ws->derrlocs; + int *errlocs = ws->errlocs; + int dlen = len - nroots; + int nn = rs->codec->nn; + uint16_t *c = ws->c; + uint16_t *r = ws->r; + int errval; + int errloc; + int i; + + /* Load c with random data and encode */ + for (i = 0; i < dlen; i++) + c[i] = prandom_u32() & nn; + + memset(c + dlen, 0, nroots * sizeof(*c)); + encode_rs16(rs, c, dlen, c + dlen, 0); + + /* Make copyand add errors and erasures */ + memcpy(r, c, len * sizeof(*r)); + memset(errlocs, 0, len * sizeof(*errlocs)); + memset(derrlocs, 0, nroots * sizeof(*derrlocs)); + + /* Generating random errors */ + for (i = 0; i < errs; i++) { + do { + /* Error value must be nonzero */ + errval = prandom_u32() & nn; + } while (errval == 0); + + do { + /* Must not choose the same location twice */ + errloc = prandom_u32() % len; + } while (errlocs[errloc] != 0); + + errlocs[errloc] = 1; + r[errloc] ^= errval; + } + + /* Generating random erasures */ + for (i = 0; i < eras; i++) { + do { + /* Must not choose the same location twice */ + errloc = prandom_u32() % len; + } while (errlocs[errloc] != 0); + + derrlocs[i] = errloc; + + if (ewsc && (prandom_u32() & 1)) { + /* Erasure with the symbol intact */ + errlocs[errloc] = 2; + } else { + /* Erasure with corrupted symbol */ + do { + /* Error value must be nonzero */ + errval = prandom_u32() & nn; + } while (errval == 0); + + errlocs[errloc] = 1; + r[errloc] ^= errval; + errs++; + } + } + + return errs; +} + +static void fix_err(uint16_t *data, int nerrs, uint16_t *corr, int *errlocs) +{ + int i; + + for (i = 0; i < nerrs; i++) + data[errlocs[i]] ^= corr[i]; +} + +static void compute_syndrome(struct rs_control *rsc, uint16_t *data, + int len, uint16_t *syn) +{ + struct rs_codec *rs = rsc->codec; + uint16_t *alpha_to = rs->alpha_to; + uint16_t *index_of = rs->index_of; + int nroots = rs->nroots; + int prim = rs->prim; + int fcr = rs->fcr; + int i, j; + + /* Calculating syndrome */ + for (i = 0; i < nroots; i++) { + syn[i] = data[0]; + for (j = 1; j < len; j++) { + if (syn[i] == 0) { + syn[i] = data[j]; + } else { + syn[i] = data[j] ^ + alpha_to[rs_modnn(rs, index_of[syn[i]] + + (fcr + i) * prim)]; + } + } + } + + /* Convert to index form */ + for (i = 0; i < nroots; i++) + syn[i] = rs->index_of[syn[i]]; +} + +/* Test up to error correction capacity */ +static void test_uc(struct rs_control *rs, int len, int errs, + int eras, int trials, struct estat *stat, + struct wspace *ws, int method) +{ + int dlen = len - rs->codec->nroots; + int *derrlocs = ws->derrlocs; + int *errlocs = ws->errlocs; + uint16_t *corr = ws->corr; + uint16_t *c = ws->c; + uint16_t *r = ws->r; + uint16_t *s = ws->s; + int derrs, nerrs; + int i, j; + + for (j = 0; j < trials; j++) { + nerrs = get_rcw_we(rs, ws, len, errs, eras); + + switch (method) { + case CORR_BUFFER: + derrs = decode_rs16(rs, r, r + dlen, dlen, + NULL, eras, derrlocs, 0, corr); + fix_err(r, derrs, corr, derrlocs); + break; + case CALLER_SYNDROME: + compute_syndrome(rs, r, len, s); + derrs = decode_rs16(rs, NULL, NULL, dlen, + s, eras, derrlocs, 0, corr); + fix_err(r, derrs, corr, derrlocs); + break; + case IN_PLACE: + derrs = decode_rs16(rs, r, r + dlen, dlen, + NULL, eras, derrlocs, 0, NULL); + break; + default: + continue; + } + + if (derrs != nerrs) + stat->irv++; + + if (method != IN_PLACE) { + for (i = 0; i < derrs; i++) { + if (errlocs[derrlocs[i]] != 1) + stat->wepos++; + } + } + + if (memcmp(r, c, len * sizeof(*r))) + stat->dwrong++; + } + stat->nwords += trials; +} + +static int ex_rs_helper(struct rs_control *rs, struct wspace *ws, + int len, int trials, int method) +{ + static const char * const desc[] = { + "Testing correction buffer interface...", + "Testing with caller provided syndrome...", + "Testing in-place interface..." + }; + + struct estat stat = {0, 0, 0, 0}; + int nroots = rs->codec->nroots; + int errs, eras, retval; + + if (v >= V_PROGRESS) + pr_info(" %s\n", desc[method]); + + for (errs = 0; errs <= nroots / 2; errs++) + for (eras = 0; eras <= nroots - 2 * errs; eras++) + test_uc(rs, len, errs, eras, trials, &stat, ws, method); + + if (v >= V_CSUMMARY) { + pr_info(" Decodes wrong: %d / %d\n", + stat.dwrong, stat.nwords); + pr_info(" Wrong return value: %d / %d\n", + stat.irv, stat.nwords); + if (method != IN_PLACE) + pr_info(" Wrong error position: %d\n", stat.wepos); + } + + retval = stat.dwrong + stat.wepos + stat.irv; + if (retval && v >= V_PROGRESS) + pr_warn(" FAIL: %d decoding failures!\n", retval); + + return retval; +} + +static int exercise_rs(struct rs_control *rs, struct wspace *ws, + int len, int trials) +{ + + int retval = 0; + int i; + + if (v >= V_PROGRESS) + pr_info("Testing up to error correction capacity...\n"); + + for (i = 0; i <= IN_PLACE; i++) + retval |= ex_rs_helper(rs, ws, len, trials, i); + + return retval; +} + +/* Tests for correct behaviour beyond error correction capacity */ +static void test_bc(struct rs_control *rs, int len, int errs, + int eras, int trials, struct bcstat *stat, + struct wspace *ws) +{ + int nroots = rs->codec->nroots; + int dlen = len - nroots; + int *derrlocs = ws->derrlocs; + uint16_t *corr = ws->corr; + uint16_t *r = ws->r; + int derrs, j; + + for (j = 0; j < trials; j++) { + get_rcw_we(rs, ws, len, errs, eras); + derrs = decode_rs16(rs, r, r + dlen, dlen, + NULL, eras, derrlocs, 0, corr); + fix_err(r, derrs, corr, derrlocs); + + if (derrs >= 0) { + stat->rsuccess++; + + /* + * We check that the returned word is actually a + * codeword. The obious way to do this would be to + * compute the syndrome, but we don't want to replicate + * that code here. However, all the codes are in + * systematic form, and therefore we can encode the + * returned word, and see whether the parity changes or + * not. + */ + memset(corr, 0, nroots * sizeof(*corr)); + encode_rs16(rs, r, dlen, corr, 0); + + if (memcmp(r + dlen, corr, nroots * sizeof(*corr))) + stat->noncw++; + } else { + stat->rfail++; + } + } + stat->nwords += trials; +} + +static int exercise_rs_bc(struct rs_control *rs, struct wspace *ws, + int len, int trials) +{ + struct bcstat stat = {0, 0, 0, 0}; + int nroots = rs->codec->nroots; + int errs, eras, cutoff; + + if (v >= V_PROGRESS) + pr_info("Testing beyond error correction capacity...\n"); + + for (errs = 1; errs <= nroots; errs++) { + eras = nroots - 2 * errs + 1; + if (eras < 0) + eras = 0; + + cutoff = nroots <= len - errs ? nroots : len - errs; + for (; eras <= cutoff; eras++) + test_bc(rs, len, errs, eras, trials, &stat, ws); + } + + if (v >= V_CSUMMARY) { + pr_info(" decoder gives up: %d / %d\n", + stat.rfail, stat.nwords); + pr_info(" decoder returns success: %d / %d\n", + stat.rsuccess, stat.nwords); + pr_info(" not a codeword: %d / %d\n", + stat.noncw, stat.rsuccess); + } + + if (stat.noncw && v >= V_PROGRESS) + pr_warn(" FAIL: %d silent failures!\n", stat.noncw); + + return stat.noncw; +} + +static int run_exercise(struct etab *e) +{ + int nn = (1 << e->symsize) - 1; + int kk = nn - e->nroots; + struct rs_control *rsc; + int retval = -ENOMEM; + int max_pad = kk - 1; + int prev_pad = -1; + struct wspace *ws; + int i; + + rsc = init_rs(e->symsize, e->genpoly, e->fcs, e->prim, e->nroots); + if (!rsc) + return retval; + + ws = alloc_ws(rsc->codec); + if (!ws) + goto err; + + retval = 0; + for (i = 0; i < ARRAY_SIZE(pad_coef); i++) { + int pad = (pad_coef[i].mult * max_pad) >> pad_coef[i].shift; + int len = nn - pad; + + if (pad == prev_pad) + continue; + + prev_pad = pad; + if (v >= V_PROGRESS) { + pr_info("Testing (%d,%d)_%d code...\n", + len, kk - pad, nn + 1); + } + + retval |= exercise_rs(rsc, ws, len, e->ntrials); + if (bc) + retval |= exercise_rs_bc(rsc, ws, len, e->ntrials); + } + + free_ws(ws); + +err: + free_rs(rsc); + return retval; +} + +static int __init test_rslib_init(void) +{ + int i, fail = 0; + + for (i = 0; Tab[i].symsize != 0 ; i++) { + int retval; + + retval = run_exercise(Tab + i); + if (retval < 0) + return -ENOMEM; + + fail |= retval; + } + + if (fail) + pr_warn("rslib: test failed\n"); + else + pr_info("rslib: test ok\n"); + + return -EAGAIN; /* Fail will directly unload the module */ +} + +static void __exit test_rslib_exit(void) +{ +} + +module_init(test_rslib_init) +module_exit(test_rslib_exit) + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Ferdinand Blomqvist"); +MODULE_DESCRIPTION("Reed-Solomon library test"); diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 54f57cd117c6..969e5400a615 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -26,9 +26,7 @@ static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) /* * First get a stable cleared mask, setting the old mask to 0. */ - do { - mask = sb->map[index].cleared; - } while (cmpxchg(&sb->map[index].cleared, mask, 0) != mask); + mask = xchg(&sb->map[index].cleared, 0); /* * Now clear the masked bits in our free word @@ -516,10 +514,8 @@ static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq) struct sbq_wait_state *ws = &sbq->ws[wake_index]; if (waitqueue_active(&ws->wait)) { - int o = atomic_read(&sbq->wake_index); - - if (wake_index != o) - atomic_cmpxchg(&sbq->wake_index, o, wake_index); + if (wake_index != atomic_read(&sbq->wake_index)) + atomic_set(&sbq->wake_index, wake_index); return ws; } diff --git a/lib/scatterlist.c b/lib/scatterlist.c index 2882d9ba6607..c2cf2c311b7d 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -179,7 +179,8 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents) * __sg_free_table - Free a previously mapped sg table * @table: The sg table header to use * @max_ents: The maximum number of entries per single scatterlist - * @skip_first_chunk: don't free the (preallocated) first scatterlist chunk + * @nents_first_chunk: Number of entries int the (preallocated) first + * scatterlist chunk, 0 means no such preallocated first chunk * @free_fn: Free function * * Description: @@ -189,9 +190,10 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents) * **/ void __sg_free_table(struct sg_table *table, unsigned int max_ents, - bool skip_first_chunk, sg_free_fn *free_fn) + unsigned int nents_first_chunk, sg_free_fn *free_fn) { struct scatterlist *sgl, *next; + unsigned curr_max_ents = nents_first_chunk ?: max_ents; if (unlikely(!table->sgl)) return; @@ -207,9 +209,9 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, * sg_size is then one less than alloc size, since the last * element is the chain pointer. */ - if (alloc_size > max_ents) { - next = sg_chain_ptr(&sgl[max_ents - 1]); - alloc_size = max_ents; + if (alloc_size > curr_max_ents) { + next = sg_chain_ptr(&sgl[curr_max_ents - 1]); + alloc_size = curr_max_ents; sg_size = alloc_size - 1; } else { sg_size = alloc_size; @@ -217,11 +219,12 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, } table->orig_nents -= sg_size; - if (skip_first_chunk) - skip_first_chunk = false; + if (nents_first_chunk) + nents_first_chunk = 0; else free_fn(sgl, alloc_size); sgl = next; + curr_max_ents = max_ents; } table->sgl = NULL; @@ -244,6 +247,8 @@ EXPORT_SYMBOL(sg_free_table); * @table: The sg table header to use * @nents: Number of entries in sg list * @max_ents: The maximum number of entries the allocator returns per call + * @nents_first_chunk: Number of entries int the (preallocated) first + * scatterlist chunk, 0 means no such preallocated chunk provided by user * @gfp_mask: GFP allocation mask * @alloc_fn: Allocator to use * @@ -260,10 +265,13 @@ EXPORT_SYMBOL(sg_free_table); **/ int __sg_alloc_table(struct sg_table *table, unsigned int nents, unsigned int max_ents, struct scatterlist *first_chunk, - gfp_t gfp_mask, sg_alloc_fn *alloc_fn) + unsigned int nents_first_chunk, gfp_t gfp_mask, + sg_alloc_fn *alloc_fn) { struct scatterlist *sg, *prv; unsigned int left; + unsigned curr_max_ents = nents_first_chunk ?: max_ents; + unsigned prv_max_ents; memset(table, 0, sizeof(*table)); @@ -279,8 +287,8 @@ int __sg_alloc_table(struct sg_table *table, unsigned int nents, do { unsigned int sg_size, alloc_size = left; - if (alloc_size > max_ents) { - alloc_size = max_ents; + if (alloc_size > curr_max_ents) { + alloc_size = curr_max_ents; sg_size = alloc_size - 1; } else sg_size = alloc_size; @@ -314,7 +322,7 @@ int __sg_alloc_table(struct sg_table *table, unsigned int nents, * If this is not the first mapping, chain previous part. */ if (prv) - sg_chain(prv, max_ents, sg); + sg_chain(prv, prv_max_ents, sg); else table->sgl = sg; @@ -325,6 +333,8 @@ int __sg_alloc_table(struct sg_table *table, unsigned int nents, sg_mark_end(&sg[sg_size - 1]); prv = sg; + prv_max_ents = curr_max_ents; + curr_max_ents = max_ents; } while (left); return 0; @@ -347,9 +357,9 @@ int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) int ret; ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, - NULL, gfp_mask, sg_kmalloc); + NULL, 0, gfp_mask, sg_kmalloc); if (unlikely(ret)) - __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree); + __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); return ret; } @@ -676,17 +686,18 @@ static bool sg_miter_get_next_page(struct sg_mapping_iter *miter) { if (!miter->__remaining) { struct scatterlist *sg; - unsigned long pgoffset; if (!__sg_page_iter_next(&miter->piter)) return false; sg = miter->piter.sg; - pgoffset = miter->piter.sg_pgoffset; - miter->__offset = pgoffset ? 0 : sg->offset; + miter->__offset = miter->piter.sg_pgoffset ? 0 : sg->offset; + miter->piter.sg_pgoffset += miter->__offset >> PAGE_SHIFT; + miter->__offset &= PAGE_SIZE - 1; miter->__remaining = sg->offset + sg->length - - (pgoffset << PAGE_SHIFT) - miter->__offset; + (miter->piter.sg_pgoffset << PAGE_SHIFT) - + miter->__offset; miter->__remaining = min_t(unsigned long, miter->__remaining, PAGE_SIZE - miter->__offset); } diff --git a/lib/sg_pool.c b/lib/sg_pool.c index cff20df2695e..db29e5c1f790 100644 --- a/lib/sg_pool.c +++ b/lib/sg_pool.c @@ -70,18 +70,27 @@ static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask) /** * sg_free_table_chained - Free a previously mapped sg table * @table: The sg table header to use - * @first_chunk: was first_chunk not NULL in sg_alloc_table_chained? + * @nents_first_chunk: size of the first_chunk SGL passed to + * sg_alloc_table_chained * * Description: * Free an sg table previously allocated and setup with * sg_alloc_table_chained(). * + * @nents_first_chunk has to be same with that same parameter passed + * to sg_alloc_table_chained(). + * **/ -void sg_free_table_chained(struct sg_table *table, bool first_chunk) +void sg_free_table_chained(struct sg_table *table, + unsigned nents_first_chunk) { - if (first_chunk && table->orig_nents <= SG_CHUNK_SIZE) + if (table->orig_nents <= nents_first_chunk) return; - __sg_free_table(table, SG_CHUNK_SIZE, first_chunk, sg_pool_free); + + if (nents_first_chunk == 1) + nents_first_chunk = 0; + + __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free); } EXPORT_SYMBOL_GPL(sg_free_table_chained); @@ -90,31 +99,41 @@ EXPORT_SYMBOL_GPL(sg_free_table_chained); * @table: The sg table header to use * @nents: Number of entries in sg list * @first_chunk: first SGL + * @nents_first_chunk: number of the SGL of @first_chunk * * Description: * Allocate and chain SGLs in an sg table. If @nents@ is larger than - * SG_CHUNK_SIZE a chained sg table will be setup. + * @nents_first_chunk a chained sg table will be setup. @first_chunk is + * ignored if nents_first_chunk <= 1 because user expects the SGL points + * non-chain SGL. * **/ int sg_alloc_table_chained(struct sg_table *table, int nents, - struct scatterlist *first_chunk) + struct scatterlist *first_chunk, unsigned nents_first_chunk) { int ret; BUG_ON(!nents); - if (first_chunk) { - if (nents <= SG_CHUNK_SIZE) { + if (first_chunk && nents_first_chunk) { + if (nents <= nents_first_chunk) { table->nents = table->orig_nents = nents; sg_init_table(table->sgl, nents); return 0; } } + /* User supposes that the 1st SGL includes real entry */ + if (nents_first_chunk <= 1) { + first_chunk = NULL; + nents_first_chunk = 0; + } + ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE, - first_chunk, GFP_ATOMIC, sg_pool_alloc); + first_chunk, nents_first_chunk, + GFP_ATOMIC, sg_pool_alloc); if (unlikely(ret)) - sg_free_table_chained(table, (bool)first_chunk); + sg_free_table_chained(table, nents_first_chunk); return ret; } EXPORT_SYMBOL_GPL(sg_alloc_table_chained); diff --git a/lib/sg_split.c b/lib/sg_split.c index 9982c63d1063..60a0babebf2e 100644 --- a/lib/sg_split.c +++ b/lib/sg_split.c @@ -176,11 +176,13 @@ int sg_split(struct scatterlist *in, const int in_mapped_nents, * The order of these 3 calls is important and should be kept. */ sg_split_phys(splitters, nb_splits); - ret = sg_calculate_split(in, in_mapped_nents, nb_splits, skip, - split_sizes, splitters, true); - if (ret < 0) - goto err; - sg_split_mapped(splitters, nb_splits); + if (in_mapped_nents) { + ret = sg_calculate_split(in, in_mapped_nents, nb_splits, skip, + split_sizes, splitters, true); + if (ret < 0) + goto err; + sg_split_mapped(splitters, nb_splits); + } for (i = 0; i < nb_splits; i++) { out[i] = splitters[i].out_sg; diff --git a/lib/show_mem.c b/lib/show_mem.c index 5c86ef4c899f..1c26c14ffbb9 100644 --- a/lib/show_mem.c +++ b/lib/show_mem.c @@ -6,7 +6,6 @@ */ #include <linux/mm.h> -#include <linux/quicklist.h> #include <linux/cma.h> void show_mem(unsigned int filter, nodemask_t *nodemask) @@ -39,10 +38,6 @@ void show_mem(unsigned int filter, nodemask_t *nodemask) #ifdef CONFIG_CMA printk("%lu pages cma reserved\n", totalcma_pages); #endif -#ifdef CONFIG_QUICKLIST - printk("%lu pages in pagetable cache\n", - quicklist_total_size()); -#endif #ifdef CONFIG_MEMORY_FAILURE printk("%lu pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages)); #endif diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c index 157d9e31f6c2..60ba93fc42ce 100644 --- a/lib/smp_processor_id.c +++ b/lib/smp_processor_id.c @@ -23,7 +23,7 @@ unsigned int check_preemption_disabled(const char *what1, const char *what2) * Kernel threads bound to a single CPU can safely use * smp_processor_id(): */ - if (cpumask_equal(¤t->cpus_allowed, cpumask_of(this_cpu))) + if (cpumask_equal(current->cpus_ptr, cpumask_of(this_cpu))) goto out; /* diff --git a/lib/sort.c b/lib/sort.c index cf408aec3733..d54cf97e9548 100644 --- a/lib/sort.c +++ b/lib/sort.c @@ -144,6 +144,18 @@ static void do_swap(void *a, void *b, size_t size, swap_func_t swap_func) swap_func(a, b, (int)size); } +typedef int (*cmp_func_t)(const void *, const void *); +typedef int (*cmp_r_func_t)(const void *, const void *, const void *); +#define _CMP_WRAPPER ((cmp_r_func_t)0L) + +static int do_cmp(const void *a, const void *b, + cmp_r_func_t cmp, const void *priv) +{ + if (cmp == _CMP_WRAPPER) + return ((cmp_func_t)(priv))(a, b); + return cmp(a, b, priv); +} + /** * parent - given the offset of the child, find the offset of the parent. * @i: the offset of the heap element whose parent is sought. Non-zero. @@ -171,12 +183,13 @@ static size_t parent(size_t i, unsigned int lsbit, size_t size) } /** - * sort - sort an array of elements + * sort_r - sort an array of elements * @base: pointer to data to sort * @num: number of elements * @size: size of each element * @cmp_func: pointer to comparison function * @swap_func: pointer to swap function or NULL + * @priv: third argument passed to comparison function * * This function does a heapsort on the given array. You may provide * a swap_func function if you need to do something more than a memory @@ -188,9 +201,10 @@ static size_t parent(size_t i, unsigned int lsbit, size_t size) * O(n*n) worst-case behavior and extra memory requirements that make * it less suitable for kernel use. */ -void sort(void *base, size_t num, size_t size, - int (*cmp_func)(const void *, const void *), - void (*swap_func)(void *, void *, int size)) +void sort_r(void *base, size_t num, size_t size, + int (*cmp_func)(const void *, const void *, const void *), + void (*swap_func)(void *, void *, int size), + const void *priv) { /* pre-scale counters for performance */ size_t n = num * size, a = (num/2) * size; @@ -238,12 +252,12 @@ void sort(void *base, size_t num, size_t size, * average, 3/4 worst-case.) */ for (b = a; c = 2*b + size, (d = c + size) < n;) - b = cmp_func(base + c, base + d) >= 0 ? c : d; + b = do_cmp(base + c, base + d, cmp_func, priv) >= 0 ? c : d; if (d == n) /* Special case last leaf with no sibling */ b = c; /* Now backtrack from "b" to the correct location for "a" */ - while (b != a && cmp_func(base + a, base + b) >= 0) + while (b != a && do_cmp(base + a, base + b, cmp_func, priv) >= 0) b = parent(b, lsbit, size); c = b; /* Where "a" belongs */ while (b != a) { /* Shift it into place */ @@ -252,4 +266,12 @@ void sort(void *base, size_t num, size_t size, } } } +EXPORT_SYMBOL(sort_r); + +void sort(void *base, size_t num, size_t size, + int (*cmp_func)(const void *, const void *), + void (*swap_func)(void *, void *, int size)) +{ + return sort_r(base, num, size, _CMP_WRAPPER, swap_func, cmp_func); +} EXPORT_SYMBOL(sort); diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 66cab785bea0..ed717dd08ff3 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -87,7 +87,7 @@ static bool init_stack_slab(void **prealloc) stack_slabs[depot_index + 1] = *prealloc; /* * This smp_store_release pairs with smp_load_acquire() from - * |next_slab_inited| above and in depot_save_stack(). + * |next_slab_inited| above and in stack_depot_save(). */ smp_store_release(&next_slab_inited, 1); } @@ -114,7 +114,7 @@ static struct stack_record *depot_alloc_stack(unsigned long *entries, int size, depot_offset = 0; /* * smp_store_release() here pairs with smp_load_acquire() from - * |next_slab_inited| in depot_save_stack() and + * |next_slab_inited| in stack_depot_save() and * init_stack_slab(). */ if (depot_index + 1 < STACK_ALLOC_MAX_SLABS) diff --git a/lib/string.c b/lib/string.c index 6016eb3ac73d..08ec58cc673b 100644 --- a/lib/string.c +++ b/lib/string.c @@ -173,8 +173,9 @@ EXPORT_SYMBOL(strlcpy); * doesn't unnecessarily force the tail of the destination buffer to be * zeroed. If zeroing is desired please use strscpy_pad(). * - * Return: The number of characters copied (not including the trailing - * %NUL) or -E2BIG if the destination buffer wasn't big enough. + * Returns: + * * The number of characters copied (not including the trailing %NUL) + * * -E2BIG if count is 0 or @src was truncated. */ ssize_t strscpy(char *dest, const char *src, size_t count) { @@ -182,7 +183,7 @@ ssize_t strscpy(char *dest, const char *src, size_t count) size_t max = count; long res = 0; - if (count == 0) + if (count == 0 || WARN_ON_ONCE(count > INT_MAX)) return -E2BIG; #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS @@ -253,8 +254,9 @@ EXPORT_SYMBOL(strscpy); * For full explanation of why you may want to consider using the * 'strscpy' functions please see the function docstring for strscpy(). * - * Return: The number of characters copied (not including the trailing - * %NUL) or -E2BIG if the destination buffer wasn't big enough. + * Returns: + * * The number of characters copied (not including the trailing %NUL) + * * -E2BIG if count is 0 or @src was truncated. */ ssize_t strscpy_pad(char *dest, const char *src, size_t count) { @@ -400,6 +402,9 @@ EXPORT_SYMBOL(strncmp); * strchr - Find the first occurrence of a character in a string * @s: The string to be searched * @c: The character to search for + * + * Note that the %NUL-terminator is considered part of the string, and can + * be searched for. */ char *strchr(const char *s, int c) { @@ -453,12 +458,18 @@ EXPORT_SYMBOL(strrchr); * @s: The string to be searched * @count: The number of characters to be searched * @c: The character to search for + * + * Note that the %NUL-terminator is considered part of the string, and can + * be searched for. */ char *strnchr(const char *s, size_t count, int c) { - for (; count-- && *s != '\0'; ++s) + while (count--) { if (*s == (char)c) return (char *)s; + if (*s++ == '\0') + break; + } return NULL; } EXPORT_SYMBOL(strnchr); @@ -737,27 +748,6 @@ void *memset(void *s, int c, size_t count) EXPORT_SYMBOL(memset); #endif -/** - * memzero_explicit - Fill a region of memory (e.g. sensitive - * keying data) with 0s. - * @s: Pointer to the start of the area. - * @count: The size of the area. - * - * Note: usually using memset() is just fine (!), but in cases - * where clearing out _local_ data at the end of a scope is - * necessary, memzero_explicit() should be used instead in - * order to prevent the compiler from optimising away zeroing. - * - * memzero_explicit() doesn't need an arch-specific version as - * it just invokes the one of memset() implicitly. - */ -void memzero_explicit(void *s, size_t count) -{ - memset(s, 0, count); - barrier_data(s); -} -EXPORT_SYMBOL(memzero_explicit); - #ifndef __HAVE_ARCH_MEMSET16 /** * memset16() - Fill a memory area with a uint16_t diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 4403e1924f73..963050c0283e 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -231,35 +231,36 @@ static bool unescape_special(char **src, char **dst) * @src: source buffer (escaped) * @dst: destination buffer (unescaped) * @size: size of the destination buffer (0 to unlimit) - * @flags: combination of the flags (bitwise OR): - * %UNESCAPE_SPACE: + * @flags: combination of the flags. + * + * Description: + * The function unquotes characters in the given string. + * + * Because the size of the output will be the same as or less than the size of + * the input, the transformation may be performed in place. + * + * Caller must provide valid source and destination pointers. Be aware that + * destination buffer will always be NULL-terminated. Source string must be + * NULL-terminated as well. The supported flags are:: + * + * UNESCAPE_SPACE: * '\f' - form feed * '\n' - new line * '\r' - carriage return * '\t' - horizontal tab * '\v' - vertical tab - * %UNESCAPE_OCTAL: + * UNESCAPE_OCTAL: * '\NNN' - byte with octal value NNN (1 to 3 digits) - * %UNESCAPE_HEX: + * UNESCAPE_HEX: * '\xHH' - byte with hexadecimal value HH (1 to 2 digits) - * %UNESCAPE_SPECIAL: + * UNESCAPE_SPECIAL: * '\"' - double quote * '\\' - backslash * '\a' - alert (BEL) * '\e' - escape - * %UNESCAPE_ANY: + * UNESCAPE_ANY: * all previous together * - * Description: - * The function unquotes characters in the given string. - * - * Because the size of the output will be the same as or less than the size of - * the input, the transformation may be performed in place. - * - * Caller must provide valid source and destination pointers. Be aware that - * destination buffer will always be NULL-terminated. Source string must be - * NULL-terminated as well. - * * Return: * The amount of the characters processed to the destination buffer excluding * trailing '\0' is returned. @@ -441,7 +442,29 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * @isz: source buffer size * @dst: destination buffer (escaped) * @osz: destination buffer size - * @flags: combination of the flags (bitwise OR): + * @flags: combination of the flags + * @only: NULL-terminated string containing characters used to limit + * the selected escape class. If characters are included in @only + * that would not normally be escaped by the classes selected + * in @flags, they will be copied to @dst unescaped. + * + * Description: + * The process of escaping byte buffer includes several parts. They are applied + * in the following sequence. + * + * 1. The character is matched to the printable class, if asked, and in + * case of match it passes through to the output. + * 2. The character is not matched to the one from @only string and thus + * must go as-is to the output. + * 3. The character is checked if it falls into the class given by @flags. + * %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any + * character. Note that they actually can't go together, otherwise + * %ESCAPE_HEX will be ignored. + * + * Caller must provide valid source and destination pointers. Be aware that + * destination buffer will not be NULL-terminated, thus caller have to append + * it if needs. The supported flags are:: + * * %ESCAPE_SPACE: (special white space, not space itself) * '\f' - form feed * '\n' - new line @@ -464,26 +487,6 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * all previous together * %ESCAPE_HEX: * '\xHH' - byte with hexadecimal value HH (2 digits) - * @only: NULL-terminated string containing characters used to limit - * the selected escape class. If characters are included in @only - * that would not normally be escaped by the classes selected - * in @flags, they will be copied to @dst unescaped. - * - * Description: - * The process of escaping byte buffer includes several parts. They are applied - * in the following sequence. - * 1. The character is matched to the printable class, if asked, and in - * case of match it passes through to the output. - * 2. The character is not matched to the one from @only string and thus - * must go as-is to the output. - * 3. The character is checked if it falls into the class given by @flags. - * %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any - * character. Note that they actually can't go together, otherwise - * %ESCAPE_HEX will be ignored. - * - * Caller must provide valid source and destination pointers. Be aware that - * destination buffer will not be NULL-terminated, thus caller have to append - * it if needs. * * Return: * The total size of the escaped output that would be generated for @@ -540,6 +543,25 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, } EXPORT_SYMBOL(string_escape_mem); +int string_escape_mem_ascii(const char *src, size_t isz, char *dst, + size_t osz) +{ + char *p = dst; + char *end = p + osz; + + while (isz--) { + unsigned char c = *src++; + + if (!isprint(c) || !isascii(c) || c == '"' || c == '\\') + escape_hex(c, &p, end); + else + escape_passthrough(c, &p, end); + } + + return p - dst; +} +EXPORT_SYMBOL(string_escape_mem_ascii); + /* * Return an allocated string that has been escaped of special characters * and double quotes, making it safe to log in quotes. diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c index 023ba9f3b99f..dccb95af6003 100644 --- a/lib/strncpy_from_user.c +++ b/lib/strncpy_from_user.c @@ -6,6 +6,7 @@ #include <linux/uaccess.h> #include <linux/kernel.h> #include <linux/errno.h> +#include <linux/mm.h> #include <asm/byteorder.h> #include <asm/word-at-a-time.h> @@ -108,7 +109,7 @@ long strncpy_from_user(char *dst, const char __user *src, long count) return 0; max_addr = user_addr_max(); - src_addr = (unsigned long)src; + src_addr = (unsigned long)untagged_addr(src); if (likely(src_addr < max_addr)) { unsigned long max = max_addr - src_addr; long retval; diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c index 7f2db3fe311f..6c0005d5dd5c 100644 --- a/lib/strnlen_user.c +++ b/lib/strnlen_user.c @@ -2,16 +2,11 @@ #include <linux/kernel.h> #include <linux/export.h> #include <linux/uaccess.h> +#include <linux/mm.h> +#include <linux/bitops.h> #include <asm/word-at-a-time.h> -/* Set bits in the first 'n' bytes when loaded from memory */ -#ifdef __LITTLE_ENDIAN -# define aligned_byte_mask(n) ((1ul << 8*(n))-1) -#else -# define aligned_byte_mask(n) (~0xfful << (BITS_PER_LONG - 8 - 8*(n))) -#endif - /* * Do a strnlen, return length of string *with* final '\0'. * 'count' is the user-supplied count, while 'max' is the @@ -109,7 +104,7 @@ long strnlen_user(const char __user *str, long count) return 0; max_addr = user_addr_max(); - src_addr = (unsigned long)str; + src_addr = (unsigned long)untagged_addr(str); if (likely(src_addr < max_addr)) { unsigned long max = max_addr - src_addr; long retval; diff --git a/lib/test_blackhole_dev.c b/lib/test_blackhole_dev.c new file mode 100644 index 000000000000..4c40580a99a3 --- /dev/null +++ b/lib/test_blackhole_dev.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This module tests the blackhole_dev that is created during the + * net subsystem initialization. The test this module performs is + * by injecting an skb into the stack with skb->dev as the + * blackhole_dev and expects kernel to behave in a sane manner + * (in other words, *not crash*)! + * + * Copyright (c) 2018, Mahesh Bandewar <maheshb@google.com> + */ + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/printk.h> +#include <linux/skbuff.h> +#include <linux/netdevice.h> +#include <linux/udp.h> +#include <linux/ipv6.h> + +#include <net/dst.h> + +#define SKB_SIZE 256 +#define HEAD_SIZE (14+40+8) /* Ether + IPv6 + UDP */ +#define TAIL_SIZE 32 /* random tail-room */ + +#define UDP_PORT 1234 + +static int __init test_blackholedev_init(void) +{ + struct ipv6hdr *ip6h; + struct sk_buff *skb; + struct ethhdr *ethh; + struct udphdr *uh; + int data_len; + int ret; + + skb = alloc_skb(SKB_SIZE, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + /* Reserve head-room for the headers */ + skb_reserve(skb, HEAD_SIZE); + + /* Add data to the skb */ + data_len = SKB_SIZE - (HEAD_SIZE + TAIL_SIZE); + memset(__skb_put(skb, data_len), 0xf, data_len); + + /* Add protocol data */ + /* (Transport) UDP */ + uh = (struct udphdr *)skb_push(skb, sizeof(struct udphdr)); + skb_set_transport_header(skb, 0); + uh->source = uh->dest = htons(UDP_PORT); + uh->len = htons(data_len); + uh->check = 0; + /* (Network) IPv6 */ + ip6h = (struct ipv6hdr *)skb_push(skb, sizeof(struct ipv6hdr)); + skb_set_network_header(skb, 0); + ip6h->hop_limit = 32; + ip6h->payload_len = data_len + sizeof(struct udphdr); + ip6h->nexthdr = IPPROTO_UDP; + ip6h->saddr = in6addr_loopback; + ip6h->daddr = in6addr_loopback; + /* Ether */ + ethh = (struct ethhdr *)skb_push(skb, sizeof(struct ethhdr)); + skb_set_mac_header(skb, 0); + + skb->protocol = htons(ETH_P_IPV6); + skb->pkt_type = PACKET_HOST; + skb->dev = blackhole_netdev; + + /* Now attempt to send the packet */ + ret = dev_queue_xmit(skb); + + switch (ret) { + case NET_XMIT_SUCCESS: + pr_warn("dev_queue_xmit() returned NET_XMIT_SUCCESS\n"); + break; + case NET_XMIT_DROP: + pr_warn("dev_queue_xmit() returned NET_XMIT_DROP\n"); + break; + case NET_XMIT_CN: + pr_warn("dev_queue_xmit() returned NET_XMIT_CN\n"); + break; + default: + pr_err("dev_queue_xmit() returned UNKNOWN(%d)\n", ret); + } + + return 0; +} + +static void __exit test_blackholedev_exit(void) +{ + pr_warn("test_blackholedev module terminating.\n"); +} + +module_init(test_blackholedev_init); +module_exit(test_blackholedev_exit); + +MODULE_AUTHOR("Mahesh Bandewar <maheshb@google.com>"); +MODULE_LICENSE("GPL"); diff --git a/lib/test_bpf.c b/lib/test_bpf.c index c41705835cba..5ef3eccee27c 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -867,7 +867,7 @@ static struct bpf_test tests[] = { }, CLASSIC, { }, - { { 4, 10 ^ 300 }, { 20, 10 ^ 300 } }, + { { 4, 0xA ^ 300 }, { 20, 0xA ^ 300 } }, }, { "SPILL_FILL", diff --git a/lib/test_firmware.c b/lib/test_firmware.c index 83ea6c4e623c..251213c872b5 100644 --- a/lib/test_firmware.c +++ b/lib/test_firmware.c @@ -18,6 +18,7 @@ #include <linux/device.h> #include <linux/fs.h> #include <linux/miscdevice.h> +#include <linux/sizes.h> #include <linux/slab.h> #include <linux/uaccess.h> #include <linux/delay.h> @@ -26,6 +27,7 @@ #define TEST_FIRMWARE_NAME "test-firmware.bin" #define TEST_FIRMWARE_NUM_REQS 4 +#define TEST_FIRMWARE_BUF_SIZE SZ_1K static DEFINE_MUTEX(test_fw_mutex); static const struct firmware *test_firmware; @@ -45,6 +47,8 @@ struct test_batched_req { * test_config - represents configuration for the test for different triggers * * @name: the name of the firmware file to look for + * @into_buf: when the into_buf is used if this is true + * request_firmware_into_buf() will be used instead. * @sync_direct: when the sync trigger is used if this is true * request_firmware_direct() will be used instead. * @send_uevent: whether or not to send a uevent for async requests @@ -83,6 +87,7 @@ struct test_batched_req { */ struct test_config { char *name; + bool into_buf; bool sync_direct; bool send_uevent; u8 num_requests; @@ -176,6 +181,7 @@ static int __test_firmware_config_init(void) test_fw_config->num_requests = TEST_FIRMWARE_NUM_REQS; test_fw_config->send_uevent = true; + test_fw_config->into_buf = false; test_fw_config->sync_direct = false; test_fw_config->req_firmware = request_firmware; test_fw_config->test_result = 0; @@ -245,6 +251,9 @@ static ssize_t config_show(struct device *dev, "FW_ACTION_HOTPLUG" : "FW_ACTION_NOHOTPLUG"); len += scnprintf(buf+len, PAGE_SIZE - len, + "into_buf:\t\t%s\n", + test_fw_config->into_buf ? "true" : "false"); + len += scnprintf(buf+len, PAGE_SIZE - len, "sync_direct:\t\t%s\n", test_fw_config->sync_direct ? "true" : "false"); len += scnprintf(buf+len, PAGE_SIZE - len, @@ -393,6 +402,23 @@ static ssize_t config_num_requests_show(struct device *dev, } static DEVICE_ATTR_RW(config_num_requests); +static ssize_t config_into_buf_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + return test_dev_config_update_bool(buf, + count, + &test_fw_config->into_buf); +} + +static ssize_t config_into_buf_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return test_dev_config_show_bool(buf, test_fw_config->into_buf); +} +static DEVICE_ATTR_RW(config_into_buf); + static ssize_t config_sync_direct_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -522,7 +548,7 @@ static ssize_t trigger_async_request_store(struct device *dev, rc = count; } else { pr_err("failed to async load firmware\n"); - rc = -ENODEV; + rc = -ENOMEM; } out: @@ -585,7 +611,26 @@ static int test_fw_run_batch_request(void *data) return -EINVAL; } - req->rc = test_fw_config->req_firmware(&req->fw, req->name, req->dev); + if (test_fw_config->into_buf) { + void *test_buf; + + test_buf = kzalloc(TEST_FIRMWARE_BUF_SIZE, GFP_KERNEL); + if (!test_buf) + return -ENOSPC; + + req->rc = request_firmware_into_buf(&req->fw, + req->name, + req->dev, + test_buf, + TEST_FIRMWARE_BUF_SIZE); + if (!req->fw) + kfree(test_buf); + } else { + req->rc = test_fw_config->req_firmware(&req->fw, + req->name, + req->dev); + } + if (req->rc) { pr_info("#%u: batched sync load failed: %d\n", req->idx, req->rc); @@ -849,6 +894,7 @@ static struct attribute *test_dev_attrs[] = { TEST_FW_DEV_ATTR(config), TEST_FW_DEV_ATTR(config_name), TEST_FW_DEV_ATTR(config_num_requests), + TEST_FW_DEV_ATTR(config_into_buf), TEST_FW_DEV_ATTR(config_sync_direct), TEST_FW_DEV_ATTR(config_send_uevent), TEST_FW_DEV_ATTR(config_read_fw_idx), @@ -886,8 +932,11 @@ static int __init test_firmware_init(void) return -ENOMEM; rc = __test_firmware_config_init(); - if (rc) + if (rc) { + kfree(test_fw_config); + pr_err("could not init firmware test config: %d\n", rc); return rc; + } rc = misc_register(&test_fw_misc_device); if (rc) { diff --git a/lib/test_kasan.c b/lib/test_kasan.c index e3c593c38eff..49cc4d570a40 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -7,16 +7,20 @@ #define pr_fmt(fmt) "kasan test: %s " fmt, __func__ +#include <linux/bitops.h> #include <linux/delay.h> +#include <linux/kasan.h> #include <linux/kernel.h> -#include <linux/mman.h> #include <linux/mm.h> +#include <linux/mman.h> +#include <linux/module.h> #include <linux/printk.h> #include <linux/slab.h> #include <linux/string.h> #include <linux/uaccess.h> -#include <linux/module.h> -#include <linux/kasan.h> +#include <linux/io.h> + +#include <asm/page.h> /* * Note: test functions are marked noinline so that their names appear in @@ -336,6 +340,42 @@ static noinline void __init kmalloc_uaf2(void) kfree(ptr2); } +static noinline void __init kfree_via_page(void) +{ + char *ptr; + size_t size = 8; + struct page *page; + unsigned long offset; + + pr_info("invalid-free false positive (via page)\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + page = virt_to_page(ptr); + offset = offset_in_page(ptr); + kfree(page_address(page) + offset); +} + +static noinline void __init kfree_via_phys(void) +{ + char *ptr; + size_t size = 8; + phys_addr_t phys; + + pr_info("invalid-free false positive (via phys)\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + phys = virt_to_phys(ptr); + kfree(phys_to_virt(phys)); +} + static noinline void __init kmem_cache_oob(void) { char *p; @@ -619,6 +659,95 @@ static noinline void __init kasan_strings(void) strnlen(ptr, 1); } +static noinline void __init kasan_bitops(void) +{ + /* + * Allocate 1 more byte, which causes kzalloc to round up to 16-bytes; + * this way we do not actually corrupt other memory. + */ + long *bits = kzalloc(sizeof(*bits) + 1, GFP_KERNEL); + if (!bits) + return; + + /* + * Below calls try to access bit within allocated memory; however, the + * below accesses are still out-of-bounds, since bitops are defined to + * operate on the whole long the bit is in. + */ + pr_info("out-of-bounds in set_bit\n"); + set_bit(BITS_PER_LONG, bits); + + pr_info("out-of-bounds in __set_bit\n"); + __set_bit(BITS_PER_LONG, bits); + + pr_info("out-of-bounds in clear_bit\n"); + clear_bit(BITS_PER_LONG, bits); + + pr_info("out-of-bounds in __clear_bit\n"); + __clear_bit(BITS_PER_LONG, bits); + + pr_info("out-of-bounds in clear_bit_unlock\n"); + clear_bit_unlock(BITS_PER_LONG, bits); + + pr_info("out-of-bounds in __clear_bit_unlock\n"); + __clear_bit_unlock(BITS_PER_LONG, bits); + + pr_info("out-of-bounds in change_bit\n"); + change_bit(BITS_PER_LONG, bits); + + pr_info("out-of-bounds in __change_bit\n"); + __change_bit(BITS_PER_LONG, bits); + + /* + * Below calls try to access bit beyond allocated memory. + */ + pr_info("out-of-bounds in test_and_set_bit\n"); + test_and_set_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + + pr_info("out-of-bounds in __test_and_set_bit\n"); + __test_and_set_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + + pr_info("out-of-bounds in test_and_set_bit_lock\n"); + test_and_set_bit_lock(BITS_PER_LONG + BITS_PER_BYTE, bits); + + pr_info("out-of-bounds in test_and_clear_bit\n"); + test_and_clear_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + + pr_info("out-of-bounds in __test_and_clear_bit\n"); + __test_and_clear_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + + pr_info("out-of-bounds in test_and_change_bit\n"); + test_and_change_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + + pr_info("out-of-bounds in __test_and_change_bit\n"); + __test_and_change_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + + pr_info("out-of-bounds in test_bit\n"); + (void)test_bit(BITS_PER_LONG + BITS_PER_BYTE, bits); + +#if defined(clear_bit_unlock_is_negative_byte) + pr_info("out-of-bounds in clear_bit_unlock_is_negative_byte\n"); + clear_bit_unlock_is_negative_byte(BITS_PER_LONG + BITS_PER_BYTE, bits); +#endif + kfree(bits); +} + +static noinline void __init kmalloc_double_kzfree(void) +{ + char *ptr; + size_t size = 16; + + pr_info("double-free (kzfree)\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + kzfree(ptr); + kzfree(ptr); +} + static int __init kmalloc_tests_init(void) { /* @@ -647,6 +776,8 @@ static int __init kmalloc_tests_init(void) kmalloc_uaf(); kmalloc_uaf_memset(); kmalloc_uaf2(); + kfree_via_page(); + kfree_via_phys(); kmem_cache_oob(); memcg_accounted_kmem_cache(); kasan_stack_oob(); @@ -660,6 +791,8 @@ static int __init kmalloc_tests_init(void) kasan_memchr(); kasan_memcmp(); kasan_strings(); + kasan_bitops(); + kmalloc_double_kzfree(); kasan_restore_multi_shot(multishot); diff --git a/lib/test_meminit.c b/lib/test_meminit.c new file mode 100644 index 000000000000..9742e5cb853a --- /dev/null +++ b/lib/test_meminit.c @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test cases for SL[AOU]B/page initialization at alloc/free time. + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/vmalloc.h> + +#define GARBAGE_INT (0x09A7BA9E) +#define GARBAGE_BYTE (0x9E) + +#define REPORT_FAILURES_IN_FN() \ + do { \ + if (failures) \ + pr_info("%s failed %d out of %d times\n", \ + __func__, failures, num_tests); \ + else \ + pr_info("all %d tests in %s passed\n", \ + num_tests, __func__); \ + } while (0) + +/* Calculate the number of uninitialized bytes in the buffer. */ +static int __init count_nonzero_bytes(void *ptr, size_t size) +{ + int i, ret = 0; + unsigned char *p = (unsigned char *)ptr; + + for (i = 0; i < size; i++) + if (p[i]) + ret++; + return ret; +} + +/* Fill a buffer with garbage, skipping |skip| first bytes. */ +static void __init fill_with_garbage_skip(void *ptr, int size, size_t skip) +{ + unsigned int *p = (unsigned int *)((char *)ptr + skip); + int i = 0; + + WARN_ON(skip > size); + size -= skip; + + while (size >= sizeof(*p)) { + p[i] = GARBAGE_INT; + i++; + size -= sizeof(*p); + } + if (size) + memset(&p[i], GARBAGE_BYTE, size); +} + +static void __init fill_with_garbage(void *ptr, size_t size) +{ + fill_with_garbage_skip(ptr, size, 0); +} + +static int __init do_alloc_pages_order(int order, int *total_failures) +{ + struct page *page; + void *buf; + size_t size = PAGE_SIZE << order; + + page = alloc_pages(GFP_KERNEL, order); + buf = page_address(page); + fill_with_garbage(buf, size); + __free_pages(page, order); + + page = alloc_pages(GFP_KERNEL, order); + buf = page_address(page); + if (count_nonzero_bytes(buf, size)) + (*total_failures)++; + fill_with_garbage(buf, size); + __free_pages(page, order); + return 1; +} + +/* Test the page allocator by calling alloc_pages with different orders. */ +static int __init test_pages(int *total_failures) +{ + int failures = 0, num_tests = 0; + int i; + + for (i = 0; i < 10; i++) + num_tests += do_alloc_pages_order(i, &failures); + + REPORT_FAILURES_IN_FN(); + *total_failures += failures; + return num_tests; +} + +/* Test kmalloc() with given parameters. */ +static int __init do_kmalloc_size(size_t size, int *total_failures) +{ + void *buf; + + buf = kmalloc(size, GFP_KERNEL); + fill_with_garbage(buf, size); + kfree(buf); + + buf = kmalloc(size, GFP_KERNEL); + if (count_nonzero_bytes(buf, size)) + (*total_failures)++; + fill_with_garbage(buf, size); + kfree(buf); + return 1; +} + +/* Test vmalloc() with given parameters. */ +static int __init do_vmalloc_size(size_t size, int *total_failures) +{ + void *buf; + + buf = vmalloc(size); + fill_with_garbage(buf, size); + vfree(buf); + + buf = vmalloc(size); + if (count_nonzero_bytes(buf, size)) + (*total_failures)++; + fill_with_garbage(buf, size); + vfree(buf); + return 1; +} + +/* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ +static int __init test_kvmalloc(int *total_failures) +{ + int failures = 0, num_tests = 0; + int i, size; + + for (i = 0; i < 20; i++) { + size = 1 << i; + num_tests += do_kmalloc_size(size, &failures); + num_tests += do_vmalloc_size(size, &failures); + } + + REPORT_FAILURES_IN_FN(); + *total_failures += failures; + return num_tests; +} + +#define CTOR_BYTES (sizeof(unsigned int)) +#define CTOR_PATTERN (0x41414141) +/* Initialize the first 4 bytes of the object. */ +static void test_ctor(void *obj) +{ + *(unsigned int *)obj = CTOR_PATTERN; +} + +/* + * Check the invariants for the buffer allocated from a slab cache. + * If the cache has a test constructor, the first 4 bytes of the object must + * always remain equal to CTOR_PATTERN. + * If the cache isn't an RCU-typesafe one, or if the allocation is done with + * __GFP_ZERO, then the object contents must be zeroed after allocation. + * If the cache is an RCU-typesafe one, the object contents must never be + * zeroed after the first use. This is checked by memcmp() in + * do_kmem_cache_size(). + */ +static bool __init check_buf(void *buf, int size, bool want_ctor, + bool want_rcu, bool want_zero) +{ + int bytes; + bool fail = false; + + bytes = count_nonzero_bytes(buf, size); + WARN_ON(want_ctor && want_zero); + if (want_zero) + return bytes; + if (want_ctor) { + if (*(unsigned int *)buf != CTOR_PATTERN) + fail = 1; + } else { + if (bytes) + fail = !want_rcu; + } + return fail; +} + +/* + * Test kmem_cache with given parameters: + * want_ctor - use a constructor; + * want_rcu - use SLAB_TYPESAFE_BY_RCU; + * want_zero - use __GFP_ZERO. + */ +static int __init do_kmem_cache_size(size_t size, bool want_ctor, + bool want_rcu, bool want_zero, + int *total_failures) +{ + struct kmem_cache *c; + int iter; + bool fail = false; + gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0); + void *buf, *buf_copy; + + c = kmem_cache_create("test_cache", size, 1, + want_rcu ? SLAB_TYPESAFE_BY_RCU : 0, + want_ctor ? test_ctor : NULL); + for (iter = 0; iter < 10; iter++) { + buf = kmem_cache_alloc(c, alloc_mask); + /* Check that buf is zeroed, if it must be. */ + fail = check_buf(buf, size, want_ctor, want_rcu, want_zero); + fill_with_garbage_skip(buf, size, want_ctor ? CTOR_BYTES : 0); + + if (!want_rcu) { + kmem_cache_free(c, buf); + continue; + } + + /* + * If this is an RCU cache, use a critical section to ensure we + * can touch objects after they're freed. + */ + rcu_read_lock(); + /* + * Copy the buffer to check that it's not wiped on + * free(). + */ + buf_copy = kmalloc(size, GFP_ATOMIC); + if (buf_copy) + memcpy(buf_copy, buf, size); + + kmem_cache_free(c, buf); + /* + * Check that |buf| is intact after kmem_cache_free(). + * |want_zero| is false, because we wrote garbage to + * the buffer already. + */ + fail |= check_buf(buf, size, want_ctor, want_rcu, + false); + if (buf_copy) { + fail |= (bool)memcmp(buf, buf_copy, size); + kfree(buf_copy); + } + rcu_read_unlock(); + } + kmem_cache_destroy(c); + + *total_failures += fail; + return 1; +} + +/* + * Check that the data written to an RCU-allocated object survives + * reallocation. + */ +static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) +{ + struct kmem_cache *c; + void *buf, *buf_contents, *saved_ptr; + void **used_objects; + int i, iter, maxiter = 1024; + bool fail = false; + + c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU, + NULL); + buf = kmem_cache_alloc(c, GFP_KERNEL); + saved_ptr = buf; + fill_with_garbage(buf, size); + buf_contents = kmalloc(size, GFP_KERNEL); + if (!buf_contents) + goto out; + used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL); + if (!used_objects) { + kfree(buf_contents); + goto out; + } + memcpy(buf_contents, buf, size); + kmem_cache_free(c, buf); + /* + * Run for a fixed number of iterations. If we never hit saved_ptr, + * assume the test passes. + */ + for (iter = 0; iter < maxiter; iter++) { + buf = kmem_cache_alloc(c, GFP_KERNEL); + used_objects[iter] = buf; + if (buf == saved_ptr) { + fail = memcmp(buf_contents, buf, size); + for (i = 0; i <= iter; i++) + kmem_cache_free(c, used_objects[i]); + goto free_out; + } + } + +free_out: + kmem_cache_destroy(c); + kfree(buf_contents); + kfree(used_objects); +out: + *total_failures += fail; + return 1; +} + +static int __init do_kmem_cache_size_bulk(int size, int *total_failures) +{ + struct kmem_cache *c; + int i, iter, maxiter = 1024; + int num, bytes; + bool fail = false; + void *objects[10]; + + c = kmem_cache_create("test_cache", size, size, 0, NULL); + for (iter = 0; (iter < maxiter) && !fail; iter++) { + num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), + objects); + for (i = 0; i < num; i++) { + bytes = count_nonzero_bytes(objects[i], size); + if (bytes) + fail = true; + fill_with_garbage(objects[i], size); + } + + if (num) + kmem_cache_free_bulk(c, num, objects); + } + *total_failures += fail; + return 1; +} + +/* + * Test kmem_cache allocation by creating caches of different sizes, with and + * without constructors, with and without SLAB_TYPESAFE_BY_RCU. + */ +static int __init test_kmemcache(int *total_failures) +{ + int failures = 0, num_tests = 0; + int i, flags, size; + bool ctor, rcu, zero; + + for (i = 0; i < 10; i++) { + size = 8 << i; + for (flags = 0; flags < 8; flags++) { + ctor = flags & 1; + rcu = flags & 2; + zero = flags & 4; + if (ctor & zero) + continue; + num_tests += do_kmem_cache_size(size, ctor, rcu, zero, + &failures); + } + num_tests += do_kmem_cache_size_bulk(size, &failures); + } + REPORT_FAILURES_IN_FN(); + *total_failures += failures; + return num_tests; +} + +/* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */ +static int __init test_rcu_persistent(int *total_failures) +{ + int failures = 0, num_tests = 0; + int i, size; + + for (i = 0; i < 10; i++) { + size = 8 << i; + num_tests += do_kmem_cache_rcu_persistent(size, &failures); + } + REPORT_FAILURES_IN_FN(); + *total_failures += failures; + return num_tests; +} + +/* + * Run the tests. Each test function returns the number of executed tests and + * updates |failures| with the number of failed tests. + */ +static int __init test_meminit_init(void) +{ + int failures = 0, num_tests = 0; + + num_tests += test_pages(&failures); + num_tests += test_kvmalloc(&failures); + num_tests += test_kmemcache(&failures); + num_tests += test_rcu_persistent(&failures); + + if (failures == 0) + pr_info("all %d tests passed!\n", num_tests); + else + pr_info("failures: %d out of %d\n", failures, num_tests); + + return failures ? -EINVAL : 0; +} +module_init(test_meminit_init); + +MODULE_LICENSE("GPL"); diff --git a/lib/test_overflow.c b/lib/test_overflow.c index fc680562d8b6..7a4b6f6c5473 100644 --- a/lib/test_overflow.c +++ b/lib/test_overflow.c @@ -486,16 +486,17 @@ static int __init test_overflow_shift(void) * Deal with the various forms of allocator arguments. See comments above * the DEFINE_TEST_ALLOC() instances for mapping of the "bits". */ -#define alloc010(alloc, arg, sz) alloc(sz, GFP_KERNEL) -#define alloc011(alloc, arg, sz) alloc(sz, GFP_KERNEL, NUMA_NO_NODE) +#define alloc_GFP (GFP_KERNEL | __GFP_NOWARN) +#define alloc010(alloc, arg, sz) alloc(sz, alloc_GFP) +#define alloc011(alloc, arg, sz) alloc(sz, alloc_GFP, NUMA_NO_NODE) #define alloc000(alloc, arg, sz) alloc(sz) #define alloc001(alloc, arg, sz) alloc(sz, NUMA_NO_NODE) -#define alloc110(alloc, arg, sz) alloc(arg, sz, GFP_KERNEL) +#define alloc110(alloc, arg, sz) alloc(arg, sz, alloc_GFP) #define free0(free, arg, ptr) free(ptr) #define free1(free, arg, ptr) free(arg, ptr) -/* Wrap around to 8K */ -#define TEST_SIZE (9 << PAGE_SHIFT) +/* Wrap around to 16K */ +#define TEST_SIZE (5 * 4096) #define DEFINE_TEST_ALLOC(func, free_func, want_arg, want_gfp, want_node)\ static int __init test_ ## func (void *arg) \ diff --git a/lib/test_printf.c b/lib/test_printf.c index 944eb50f3862..5d94cbff2120 100644 --- a/lib/test_printf.c +++ b/lib/test_printf.c @@ -455,6 +455,11 @@ dentry(void) test("foo", "%pd", &test_dentry[0]); test("foo", "%pd2", &test_dentry[0]); + test("(null)", "%pd", NULL); + test("(efault)", "%pd", PTR_INVALID); + test("(null)", "%pD", NULL); + test("(efault)", "%pD", PTR_INVALID); + test("romeo", "%pd", &test_dentry[3]); test("alfa/romeo", "%pd2", &test_dentry[3]); test("bravo/alfa/romeo", "%pd3", &test_dentry[3]); diff --git a/lib/test_string.c b/lib/test_string.c index bf8def01ed20..7b31f4a505bf 100644 --- a/lib/test_string.c +++ b/lib/test_string.c @@ -36,7 +36,7 @@ static __init int memset16_selftest(void) fail: kfree(p); if (i < 256) - return (i << 24) | (j << 16) | k; + return (i << 24) | (j << 16) | k | 0x8000; return 0; } @@ -72,7 +72,7 @@ static __init int memset32_selftest(void) fail: kfree(p); if (i < 256) - return (i << 24) | (j << 16) | k; + return (i << 24) | (j << 16) | k | 0x8000; return 0; } @@ -108,7 +108,74 @@ static __init int memset64_selftest(void) fail: kfree(p); if (i < 256) - return (i << 24) | (j << 16) | k; + return (i << 24) | (j << 16) | k | 0x8000; + return 0; +} + +static __init int strchr_selftest(void) +{ + const char *test_string = "abcdefghijkl"; + const char *empty_string = ""; + char *result; + int i; + + for (i = 0; i < strlen(test_string) + 1; i++) { + result = strchr(test_string, test_string[i]); + if (result - test_string != i) + return i + 'a'; + } + + result = strchr(empty_string, '\0'); + if (result != empty_string) + return 0x101; + + result = strchr(empty_string, 'a'); + if (result) + return 0x102; + + result = strchr(test_string, 'z'); + if (result) + return 0x103; + + return 0; +} + +static __init int strnchr_selftest(void) +{ + const char *test_string = "abcdefghijkl"; + const char *empty_string = ""; + char *result; + int i, j; + + for (i = 0; i < strlen(test_string) + 1; i++) { + for (j = 0; j < strlen(test_string) + 2; j++) { + result = strnchr(test_string, j, test_string[i]); + if (j <= i) { + if (!result) + continue; + return ((i + 'a') << 8) | j; + } + if (result - test_string != i) + return ((i + 'a') << 8) | j; + } + } + + result = strnchr(empty_string, 0, '\0'); + if (result) + return 0x10001; + + result = strnchr(empty_string, 1, '\0'); + if (result != empty_string) + return 0x10002; + + result = strnchr(empty_string, 1, 'a'); + if (result) + return 0x10003; + + result = strnchr(NULL, 0, '\0'); + if (result) + return 0x10004; + return 0; } @@ -131,6 +198,16 @@ static __init int string_selftest_init(void) if (subtest) goto fail; + test = 4; + subtest = strchr_selftest(); + if (subtest) + goto fail; + + test = 5; + subtest = strnchr_selftest(); + if (subtest) + goto fail; + pr_info("String selftests succeeded\n"); return 0; fail: diff --git a/lib/test_user_copy.c b/lib/test_user_copy.c index 67bcd5dfd847..5ff04d8fe971 100644 --- a/lib/test_user_copy.c +++ b/lib/test_user_copy.c @@ -31,14 +31,152 @@ # define TEST_U64 #endif -#define test(condition, msg) \ -({ \ - int cond = (condition); \ - if (cond) \ - pr_warn("%s\n", msg); \ - cond; \ +#define test(condition, msg, ...) \ +({ \ + int cond = (condition); \ + if (cond) \ + pr_warn("[%d] " msg "\n", __LINE__, ##__VA_ARGS__); \ + cond; \ }) +static bool is_zeroed(void *from, size_t size) +{ + return memchr_inv(from, 0x0, size) == NULL; +} + +static int test_check_nonzero_user(char *kmem, char __user *umem, size_t size) +{ + int ret = 0; + size_t start, end, i, zero_start, zero_end; + + if (test(size < 2 * PAGE_SIZE, "buffer too small")) + return -EINVAL; + + /* + * We want to cross a page boundary to exercise the code more + * effectively. We also don't want to make the size we scan too large, + * otherwise the test can take a long time and cause soft lockups. So + * scan a 1024 byte region across the page boundary. + */ + size = 1024; + start = PAGE_SIZE - (size / 2); + + kmem += start; + umem += start; + + zero_start = size / 4; + zero_end = size - zero_start; + + /* + * We conduct a series of check_nonzero_user() tests on a block of + * memory with the following byte-pattern (trying every possible + * [start,end] pair): + * + * [ 00 ff 00 ff ... 00 00 00 00 ... ff 00 ff 00 ] + * + * And we verify that check_nonzero_user() acts identically to + * memchr_inv(). + */ + + memset(kmem, 0x0, size); + for (i = 1; i < zero_start; i += 2) + kmem[i] = 0xff; + for (i = zero_end; i < size; i += 2) + kmem[i] = 0xff; + + ret |= test(copy_to_user(umem, kmem, size), + "legitimate copy_to_user failed"); + + for (start = 0; start <= size; start++) { + for (end = start; end <= size; end++) { + size_t len = end - start; + int retval = check_zeroed_user(umem + start, len); + int expected = is_zeroed(kmem + start, len); + + ret |= test(retval != expected, + "check_nonzero_user(=%d) != memchr_inv(=%d) mismatch (start=%zu, end=%zu)", + retval, expected, start, end); + } + } + + return ret; +} + +static int test_copy_struct_from_user(char *kmem, char __user *umem, + size_t size) +{ + int ret = 0; + char *umem_src = NULL, *expected = NULL; + size_t ksize, usize; + + umem_src = kmalloc(size, GFP_KERNEL); + ret = test(umem_src == NULL, "kmalloc failed"); + if (ret) + goto out_free; + + expected = kmalloc(size, GFP_KERNEL); + ret = test(expected == NULL, "kmalloc failed"); + if (ret) + goto out_free; + + /* Fill umem with a fixed byte pattern. */ + memset(umem_src, 0x3e, size); + ret |= test(copy_to_user(umem, umem_src, size), + "legitimate copy_to_user failed"); + + /* Check basic case -- (usize == ksize). */ + ksize = size; + usize = size; + + memcpy(expected, umem_src, ksize); + + memset(kmem, 0x0, size); + ret |= test(copy_struct_from_user(kmem, ksize, umem, usize), + "copy_struct_from_user(usize == ksize) failed"); + ret |= test(memcmp(kmem, expected, ksize), + "copy_struct_from_user(usize == ksize) gives unexpected copy"); + + /* Old userspace case -- (usize < ksize). */ + ksize = size; + usize = size / 2; + + memcpy(expected, umem_src, usize); + memset(expected + usize, 0x0, ksize - usize); + + memset(kmem, 0x0, size); + ret |= test(copy_struct_from_user(kmem, ksize, umem, usize), + "copy_struct_from_user(usize < ksize) failed"); + ret |= test(memcmp(kmem, expected, ksize), + "copy_struct_from_user(usize < ksize) gives unexpected copy"); + + /* New userspace (-E2BIG) case -- (usize > ksize). */ + ksize = size / 2; + usize = size; + + memset(kmem, 0x0, size); + ret |= test(copy_struct_from_user(kmem, ksize, umem, usize) != -E2BIG, + "copy_struct_from_user(usize > ksize) didn't give E2BIG"); + + /* New userspace (success) case -- (usize > ksize). */ + ksize = size / 2; + usize = size; + + memcpy(expected, umem_src, ksize); + ret |= test(clear_user(umem + ksize, usize - ksize), + "legitimate clear_user failed"); + + memset(kmem, 0x0, size); + ret |= test(copy_struct_from_user(kmem, ksize, umem, usize), + "copy_struct_from_user(usize > ksize) failed"); + ret |= test(memcmp(kmem, expected, ksize), + "copy_struct_from_user(usize > ksize) gives unexpected copy"); + +out_free: + kfree(expected); + kfree(umem_src); + return ret; +} + static int __init test_user_copy_init(void) { int ret = 0; @@ -106,6 +244,11 @@ static int __init test_user_copy_init(void) #endif #undef test_legit + /* Test usage of check_nonzero_user(). */ + ret |= test_check_nonzero_user(kmem, usermem, 2 * PAGE_SIZE); + /* Test usage of copy_struct_from_user(). */ + ret |= test_copy_struct_from_user(kmem, usermem, 2 * PAGE_SIZE); + /* * Invalid usage: none of these copies should succeed. */ diff --git a/lib/textsearch.c b/lib/textsearch.c index 4f16eec5d554..f68dea8806be 100644 --- a/lib/textsearch.c +++ b/lib/textsearch.c @@ -89,9 +89,9 @@ * goto errout; * } * - * pos = textsearch_find_continuous(conf, \&state, example, strlen(example)); + * pos = textsearch_find_continuous(conf, &state, example, strlen(example)); * if (pos != UINT_MAX) - * panic("Oh my god, dancing chickens at \%d\n", pos); + * panic("Oh my god, dancing chickens at %d\n", pos); * * textsearch_destroy(conf); */ diff --git a/lib/timerqueue.c b/lib/timerqueue.c index bc7e64df27df..c52710964593 100644 --- a/lib/timerqueue.c +++ b/lib/timerqueue.c @@ -26,9 +26,10 @@ */ bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node) { - struct rb_node **p = &head->head.rb_node; + struct rb_node **p = &head->rb_root.rb_root.rb_node; struct rb_node *parent = NULL; - struct timerqueue_node *ptr; + struct timerqueue_node *ptr; + bool leftmost = true; /* Make sure we don't add nodes that are already added */ WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node)); @@ -36,19 +37,17 @@ bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node) while (*p) { parent = *p; ptr = rb_entry(parent, struct timerqueue_node, node); - if (node->expires < ptr->expires) + if (node->expires < ptr->expires) { p = &(*p)->rb_left; - else + } else { p = &(*p)->rb_right; + leftmost = false; + } } rb_link_node(&node->node, parent, p); - rb_insert_color(&node->node, &head->head); + rb_insert_color_cached(&node->node, &head->rb_root, leftmost); - if (!head->next || node->expires < head->next->expires) { - head->next = node; - return true; - } - return false; + return leftmost; } EXPORT_SYMBOL_GPL(timerqueue_add); @@ -65,15 +64,10 @@ bool timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node) { WARN_ON_ONCE(RB_EMPTY_NODE(&node->node)); - /* update next pointer */ - if (head->next == node) { - struct rb_node *rbn = rb_next(&node->node); - - head->next = rb_entry_safe(rbn, struct timerqueue_node, node); - } - rb_erase(&node->node, &head->head); + rb_erase_cached(&node->node, &head->rb_root); RB_CLEAR_NODE(&node->node); - return head->next != NULL; + + return !RB_EMPTY_ROOT(&head->rb_root.rb_root); } EXPORT_SYMBOL_GPL(timerqueue_del); diff --git a/lib/usercopy.c b/lib/usercopy.c index c2bfbcaeb3dc..cbb4d9ec00f2 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/uaccess.h> +#include <linux/bitops.h> /* out-of-line parts */ @@ -31,3 +32,57 @@ unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) } EXPORT_SYMBOL(_copy_to_user); #endif + +/** + * check_zeroed_user: check if a userspace buffer only contains zero bytes + * @from: Source address, in userspace. + * @size: Size of buffer. + * + * This is effectively shorthand for "memchr_inv(from, 0, size) == NULL" for + * userspace addresses (and is more efficient because we don't care where the + * first non-zero byte is). + * + * Returns: + * * 0: There were non-zero bytes present in the buffer. + * * 1: The buffer was full of zero bytes. + * * -EFAULT: access to userspace failed. + */ +int check_zeroed_user(const void __user *from, size_t size) +{ + unsigned long val; + uintptr_t align = (uintptr_t) from % sizeof(unsigned long); + + if (unlikely(size == 0)) + return 1; + + from -= align; + size += align; + + if (!user_access_begin(from, size)) + return -EFAULT; + + unsafe_get_user(val, (unsigned long __user *) from, err_fault); + if (align) + val &= ~aligned_byte_mask(align); + + while (size > sizeof(unsigned long)) { + if (unlikely(val)) + goto done; + + from += sizeof(unsigned long); + size -= sizeof(unsigned long); + + unsafe_get_user(val, (unsigned long __user *) from, err_fault); + } + + if (size < sizeof(unsigned long)) + val &= aligned_byte_mask(size); + +done: + user_access_end(); + return (val == 0); +err_fault: + user_access_end(); + return -EFAULT; +} +EXPORT_SYMBOL(check_zeroed_user); diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig new file mode 100644 index 000000000000..9fe698ff62ec --- /dev/null +++ b/lib/vdso/Kconfig @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0 + +config HAVE_GENERIC_VDSO + bool + +if HAVE_GENERIC_VDSO + +config GENERIC_GETTIMEOFDAY + bool + help + This is a generic implementation of gettimeofday vdso. + Each architecture that enables this feature has to + provide the fallback implementation. + +config GENERIC_VDSO_32 + bool + depends on GENERIC_GETTIMEOFDAY && !64BIT + help + This config option helps to avoid possible performance issues + in 32 bit only architectures. + +config GENERIC_COMPAT_VDSO + bool + help + This config option enables the compat VDSO layer. + +endif diff --git a/lib/vdso/Makefile b/lib/vdso/Makefile new file mode 100644 index 000000000000..c415a685d61b --- /dev/null +++ b/lib/vdso/Makefile @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0 + +GENERIC_VDSO_MK_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +GENERIC_VDSO_DIR := $(dir $(GENERIC_VDSO_MK_PATH)) + +c-gettimeofday-$(CONFIG_GENERIC_GETTIMEOFDAY) := $(addprefix $(GENERIC_VDSO_DIR), gettimeofday.c) + +# This cmd checks that the vdso library does not contain absolute relocation +# It has to be called after the linking of the vdso library and requires it +# as a parameter. +# +# $(ARCH_REL_TYPE_ABS) is defined in the arch specific makefile and corresponds +# to the absolute relocation types printed by "objdump -R" and accepted by the +# dynamic linker. +ifndef ARCH_REL_TYPE_ABS +$(error ARCH_REL_TYPE_ABS is not set) +endif + +quiet_cmd_vdso_check = VDSOCHK $@ + cmd_vdso_check = if $(OBJDUMP) -R $@ | egrep -h "$(ARCH_REL_TYPE_ABS)"; \ + then (echo >&2 "$@: dynamic relocations are not supported"; \ + rm -f $@; /bin/false); fi diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c new file mode 100644 index 000000000000..45f57fd2db64 --- /dev/null +++ b/lib/vdso/gettimeofday.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Generic userspace implementations of gettimeofday() and similar. + */ +#include <linux/compiler.h> +#include <linux/math64.h> +#include <linux/time.h> +#include <linux/kernel.h> +#include <linux/hrtimer_defs.h> +#include <vdso/datapage.h> +#include <vdso/helpers.h> + +/* + * The generic vDSO implementation requires that gettimeofday.h + * provides: + * - __arch_get_vdso_data(): to get the vdso datapage. + * - __arch_get_hw_counter(): to get the hw counter based on the + * clock_mode. + * - gettimeofday_fallback(): fallback for gettimeofday. + * - clock_gettime_fallback(): fallback for clock_gettime. + * - clock_getres_fallback(): fallback for clock_getres. + */ +#ifdef ENABLE_COMPAT_VDSO +#include <asm/vdso/compat_gettimeofday.h> +#else +#include <asm/vdso/gettimeofday.h> +#endif /* ENABLE_COMPAT_VDSO */ + +#ifndef vdso_calc_delta +/* + * Default implementation which works for all sane clocksources. That + * obviously excludes x86/TSC. + */ +static __always_inline +u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) +{ + return ((cycles - last) & mask) * mult; +} +#endif + +static int do_hres(const struct vdso_data *vd, clockid_t clk, + struct __kernel_timespec *ts) +{ + const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; + u64 cycles, last, sec, ns; + u32 seq; + + do { + seq = vdso_read_begin(vd); + cycles = __arch_get_hw_counter(vd->clock_mode); + ns = vdso_ts->nsec; + last = vd->cycle_last; + if (unlikely((s64)cycles < 0)) + return -1; + + ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); + ns >>= vd->shift; + sec = vdso_ts->sec; + } while (unlikely(vdso_read_retry(vd, seq))); + + /* + * Do this outside the loop: a race inside the loop could result + * in __iter_div_u64_rem() being extremely slow. + */ + ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); + ts->tv_nsec = ns; + + return 0; +} + +static void do_coarse(const struct vdso_data *vd, clockid_t clk, + struct __kernel_timespec *ts) +{ + const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; + u32 seq; + + do { + seq = vdso_read_begin(vd); + ts->tv_sec = vdso_ts->sec; + ts->tv_nsec = vdso_ts->nsec; + } while (unlikely(vdso_read_retry(vd, seq))); +} + +static __maybe_unused int +__cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) +{ + const struct vdso_data *vd = __arch_get_vdso_data(); + u32 msk; + + /* Check for negative values or invalid clocks */ + if (unlikely((u32) clock >= MAX_CLOCKS)) + return -1; + + /* + * Convert the clockid to a bitmask and use it to check which + * clocks are handled in the VDSO directly. + */ + msk = 1U << clock; + if (likely(msk & VDSO_HRES)) { + return do_hres(&vd[CS_HRES_COARSE], clock, ts); + } else if (msk & VDSO_COARSE) { + do_coarse(&vd[CS_HRES_COARSE], clock, ts); + return 0; + } else if (msk & VDSO_RAW) { + return do_hres(&vd[CS_RAW], clock, ts); + } + return -1; +} + +static __maybe_unused int +__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + int ret = __cvdso_clock_gettime_common(clock, ts); + + if (unlikely(ret)) + return clock_gettime_fallback(clock, ts); + return 0; +} + +static __maybe_unused int +__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) +{ + struct __kernel_timespec ts; + int ret; + + ret = __cvdso_clock_gettime_common(clock, &ts); + +#ifdef VDSO_HAS_32BIT_FALLBACK + if (unlikely(ret)) + return clock_gettime32_fallback(clock, res); +#else + if (unlikely(ret)) + ret = clock_gettime_fallback(clock, &ts); +#endif + + if (likely(!ret)) { + res->tv_sec = ts.tv_sec; + res->tv_nsec = ts.tv_nsec; + } + return ret; +} + +static __maybe_unused int +__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + const struct vdso_data *vd = __arch_get_vdso_data(); + + if (likely(tv != NULL)) { + struct __kernel_timespec ts; + + if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) + return gettimeofday_fallback(tv, tz); + + tv->tv_sec = ts.tv_sec; + tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; + } + + if (unlikely(tz != NULL)) { + tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; + tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; + } + + return 0; +} + +#ifdef VDSO_HAS_TIME +static __maybe_unused time_t __cvdso_time(time_t *time) +{ + const struct vdso_data *vd = __arch_get_vdso_data(); + time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); + + if (time) + *time = t; + + return t; +} +#endif /* VDSO_HAS_TIME */ + +#ifdef VDSO_HAS_CLOCK_GETRES +static __maybe_unused +int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) +{ + const struct vdso_data *vd = __arch_get_vdso_data(); + u64 hrtimer_res; + u32 msk; + u64 ns; + + /* Check for negative values or invalid clocks */ + if (unlikely((u32) clock >= MAX_CLOCKS)) + return -1; + + hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); + /* + * Convert the clockid to a bitmask and use it to check which + * clocks are handled in the VDSO directly. + */ + msk = 1U << clock; + if (msk & VDSO_HRES) { + /* + * Preserves the behaviour of posix_get_hrtimer_res(). + */ + ns = hrtimer_res; + } else if (msk & VDSO_COARSE) { + /* + * Preserves the behaviour of posix_get_coarse_res(). + */ + ns = LOW_RES_NSEC; + } else if (msk & VDSO_RAW) { + /* + * Preserves the behaviour of posix_get_hrtimer_res(). + */ + ns = hrtimer_res; + } else { + return -1; + } + + if (likely(res)) { + res->tv_sec = 0; + res->tv_nsec = ns; + } + return 0; +} + +int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) +{ + int ret = __cvdso_clock_getres_common(clock, res); + + if (unlikely(ret)) + return clock_getres_fallback(clock, res); + return 0; +} + +static __maybe_unused int +__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) +{ + struct __kernel_timespec ts; + int ret; + + ret = __cvdso_clock_getres_common(clock, &ts); + +#ifdef VDSO_HAS_32BIT_FALLBACK + if (unlikely(ret)) + return clock_getres32_fallback(clock, res); +#else + if (unlikely(ret)) + ret = clock_getres_fallback(clock, &ts); +#endif + + if (likely(!ret && res)) { + res->tv_sec = ts.tv_sec; + res->tv_nsec = ts.tv_nsec; + } + return ret; +} +#endif /* VDSO_HAS_CLOCK_GETRES */ diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 63937044c57d..e78017a3e1bd 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -599,7 +599,7 @@ static char *string_nocheck(char *buf, char *end, const char *s, struct printf_spec spec) { int len = 0; - size_t lim = spec.precision; + int lim = spec.precision; while (lim--) { char c = *s++; @@ -869,6 +869,15 @@ char *dentry_name(char *buf, char *end, const struct dentry *d, struct printf_sp return widen_string(buf, n, end, spec); } +static noinline_for_stack +char *file_dentry_name(char *buf, char *end, const struct file *f, + struct printf_spec spec, const char *fmt) +{ + if (check_pointer(&buf, end, f, spec)) + return buf; + + return dentry_name(buf, end, f->f_path.dentry, spec, fmt); +} #ifdef CONFIG_BLOCK static noinline_for_stack char *bdev_name(char *buf, char *end, struct block_device *bdev, @@ -1799,7 +1808,7 @@ char *clock(char *buf, char *end, struct clk *clk, struct printf_spec spec, #ifdef CONFIG_COMMON_CLK return string(buf, end, __clk_get_name(clk), spec); #else - return error_string(buf, end, "(%pC?)", spec); + return ptr_to_id(buf, end, clk, spec); #endif } } @@ -2166,9 +2175,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, case 'C': return clock(buf, end, ptr, spec, fmt); case 'D': - return dentry_name(buf, end, - ((const struct file *)ptr)->f_path.dentry, - spec, fmt); + return file_dentry_name(buf, end, ptr, spec, fmt); #ifdef CONFIG_BLOCK case 'g': return bdev_name(buf, end, ptr, spec, fmt); diff --git a/lib/zstd/mem.h b/lib/zstd/mem.h index 3a0f34c8706c..93d7a2c377fe 100644 --- a/lib/zstd/mem.h +++ b/lib/zstd/mem.h @@ -27,7 +27,7 @@ /*-**************************************** * Compiler specifics ******************************************/ -#define ZSTD_STATIC static __inline __attribute__((unused)) +#define ZSTD_STATIC static inline /*-************************************************************** * Basic Types |