diff --git a/arch/arm64/kernel/cpu-reset.h b/arch/arm64/kernel/cpu-reset.h index f300e7577688..85fa1d180f85 100644 --- a/arch/arm64/kernel/cpu-reset.h +++ b/arch/arm64/kernel/cpu-reset.h @@ -16,7 +16,7 @@ void __cpu_soft_restart(unsigned long el2_switch, unsigned long entry, unsigned long arg0, unsigned long arg1, unsigned long arg2); -static inline void __noreturn cpu_soft_restart(unsigned long entry, +static inline void __noreturn __nocfi cpu_soft_restart(unsigned long entry, unsigned long arg0, unsigned long arg1, unsigned long arg2) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 71f460eb867e..c955edef232a 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -982,12 +982,8 @@ bool arch_bpf_jit_check_func(const struct bpf_prog *prog) { const uintptr_t func = (const uintptr_t)prog->bpf_func; - /* - * bpf_func must be correctly aligned and within the correct region. - */ - if (unlikely(!IS_ALIGNED(func, sizeof(u32)))) - return false; - - return (func >= BPF_JIT_REGION_START && func < BPF_JIT_REGION_END); + /* bpf_func must be correctly aligned and within the BPF JIT region */ + return (func >= BPF_JIT_REGION_START && func < BPF_JIT_REGION_END && + IS_ALIGNED(func, sizeof(u32))); } #endif diff --git a/include/linux/cfi.h b/include/linux/cfi.h index e27033d5dd53..6ef37b35faaa 100644 --- a/include/linux/cfi.h +++ b/include/linux/cfi.h @@ -1,3 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Clang Control Flow Integrity (CFI) support. + * + * Copyright (C) 2019 Google LLC + */ #ifndef _LINUX_CFI_H #define _LINUX_CFI_H @@ -6,13 +12,13 @@ #ifdef CONFIG_CFI_CLANG #ifdef CONFIG_MODULES -typedef void (*cfi_check_fn)(uint64_t, void *, void *); +typedef void (*cfi_check_fn)(uint64_t id, void *ptr, void *diag); /* Compiler-generated function in each module, and the kernel */ #define CFI_CHECK_FN __cfi_check #define CFI_CHECK_FN_NAME __stringify(CFI_CHECK_FN) -extern void CFI_CHECK_FN(uint64_t, void *, void *); +extern void CFI_CHECK_FN(uint64_t id, void *ptr, void *diag); #ifdef CONFIG_CFI_CLANG_SHADOW extern void cfi_module_add(struct module *mod, unsigned long min_addr, diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index a89045d3c403..08866a9ad037 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -49,7 +49,7 @@ __attribute__((__assume_aligned__(a, ## __VA_ARGS__))) #ifdef CONFIG_CFI_CLANG -#define __nocfi __attribute__((no_sanitize("cfi"))) +#define __nocfi __attribute__((__no_sanitize__("cfi"))) #endif #ifdef CONFIG_LTO_CLANG diff --git a/kernel/cfi.c b/kernel/cfi.c index 5092eaa403c7..31419747b86d 100644 --- a/kernel/cfi.c +++ b/kernel/cfi.c @@ -1,16 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 /* - * CFI (Control Flow Integrity) error and slowpath handling + * Clang Control Flow Integrity (CFI) error and slowpath handling. * - * Copyright (C) 2017 Google, Inc. + * Copyright (C) 2019 Google LLC */ #include +#include #include +#include #include #include #include -#include -#include #include #include @@ -25,12 +26,10 @@ static inline void handle_cfi_failure(void *ptr) { -#ifdef CONFIG_CFI_PERMISSIVE - WARN_RATELIMIT(1, "CFI failure (target: %pF):\n", ptr); -#else - pr_err("CFI failure (target: %pF):\n", ptr); - BUG(); -#endif + if (IS_ENABLED(CONFIG_CFI_PERMISSIVE)) + WARN_RATELIMIT(1, "CFI failure (target: %pS):\n", ptr); + else + panic("CFI failure (target: %pS)\n", ptr); } #ifdef CONFIG_MODULES @@ -44,7 +43,7 @@ struct shadow_range { unsigned long max_page; }; -#define SHADOW_ORDER 1 +#define SHADOW_ORDER 2 #define SHADOW_PAGES (1 << SHADOW_ORDER) #define SHADOW_SIZE \ ((SHADOW_PAGES * PAGE_SIZE - sizeof(struct shadow_range)) / sizeof(u16)) @@ -57,8 +56,8 @@ struct cfi_shadow { u16 shadow[SHADOW_SIZE]; }; -static DEFINE_SPINLOCK(shadow_update_lock); -static struct cfi_shadow __rcu *cfi_shadow __read_mostly = NULL; +static DEFINE_MUTEX(shadow_update_lock); +static struct cfi_shadow __rcu *cfi_shadow __read_mostly; static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr) { @@ -79,7 +78,8 @@ static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr) static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s, int index) { - BUG_ON(index < 0 || index >= SHADOW_SIZE); + if (unlikely(index < 0 || index >= SHADOW_SIZE)) + return 0; if (unlikely(s->shadow[index] == SHADOW_INVALID)) return 0; @@ -90,7 +90,8 @@ static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s, static inline unsigned long shadow_to_page(const struct cfi_shadow *s, int index) { - BUG_ON(index < 0 || index >= SHADOW_SIZE); + if (unlikely(index < 0 || index >= SHADOW_SIZE)) + return 0; return (s->r.min_page + index) << PAGE_SHIFT; } @@ -138,7 +139,8 @@ static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod) unsigned long check = (unsigned long)mod->cfi_check; int check_index = ptr_to_shadow(s, check); - BUG_ON((check & PAGE_MASK) != check); /* Must be page aligned */ + if (unlikely((check & PAGE_MASK) != check)) + return; /* Must be page aligned */ if (check_index < 0) return; /* Module not addressable with shadow */ @@ -151,9 +153,10 @@ static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod) /* For each page, store the check function index in the shadow */ for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) { int index = ptr_to_shadow(s, ptr); + if (index >= 0) { - /* Assume a page only contains code for one module */ - BUG_ON(s->shadow[index] != SHADOW_INVALID); + /* Each page must only contain one module */ + WARN_ON(s->shadow[index] != SHADOW_INVALID); s->shadow[index] = (u16)check_index; } } @@ -172,6 +175,7 @@ static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod) for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) { int index = ptr_to_shadow(s, ptr); + if (index >= 0) s->shadow[index] = SHADOW_INVALID; } @@ -186,14 +190,12 @@ static void update_shadow(struct module *mod, unsigned long min_addr, struct cfi_shadow *next = (struct cfi_shadow *) __get_free_pages(GFP_KERNEL, SHADOW_ORDER); - BUG_ON(!next); - next->r.mod_min_addr = min_addr; next->r.mod_max_addr = max_addr; next->r.min_page = min_addr >> PAGE_SHIFT; next->r.max_page = max_addr >> PAGE_SHIFT; - spin_lock(&shadow_update_lock); + mutex_lock(&shadow_update_lock); prev = rcu_dereference_protected(cfi_shadow, 1); prepare_next_shadow(prev, next); @@ -201,7 +203,7 @@ static void update_shadow(struct module *mod, unsigned long min_addr, set_memory_ro((unsigned long)next, SHADOW_PAGES); rcu_assign_pointer(cfi_shadow, next); - spin_unlock(&shadow_update_lock); + mutex_unlock(&shadow_update_lock); synchronize_rcu(); if (prev) { @@ -245,33 +247,36 @@ static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s, static inline cfi_check_fn find_module_cfi_check(void *ptr) { + cfi_check_fn f = CFI_CHECK_FN; struct module *mod; preempt_disable(); mod = __module_address((unsigned long)ptr); + if (mod) + f = mod->cfi_check; preempt_enable(); - if (mod) - return mod->cfi_check; - - return CFI_CHECK_FN; + return f; } static inline cfi_check_fn find_cfi_check(void *ptr) { -#ifdef CONFIG_CFI_CLANG_SHADOW + bool rcu; cfi_check_fn f; - if (!rcu_access_pointer(cfi_shadow)) - return CFI_CHECK_FN; /* No loaded modules */ + rcu = rcu_is_watching(); + if (!rcu) + rcu_nmi_enter(); +#ifdef CONFIG_CFI_CLANG_SHADOW /* Look up the __cfi_check function to use */ - rcu_read_lock(); - f = ptr_to_check_fn(rcu_dereference(cfi_shadow), (unsigned long)ptr); - rcu_read_unlock(); + rcu_read_lock_sched(); + f = ptr_to_check_fn(rcu_dereference_sched(cfi_shadow), + (unsigned long)ptr); + rcu_read_unlock_sched(); if (f) - return f; + goto out; /* * Fall back to find_module_cfi_check, which works also for a larger @@ -279,7 +284,13 @@ static inline cfi_check_fn find_cfi_check(void *ptr) */ #endif /* CONFIG_CFI_CLANG_SHADOW */ - return find_module_cfi_check(ptr); + f = find_module_cfi_check(ptr); + +out: + if (!rcu) + rcu_nmi_exit(); + + return f; } void cfi_slowpath_handler(uint64_t id, void *ptr, void *diag) diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index b8ec1dbd1be5..dfbdb3f344d0 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -158,6 +158,26 @@ static unsigned long kallsyms_sym_address(int idx) return kallsyms_relative_base - 1 - kallsyms_offsets[idx]; } +#if defined(CONFIG_CFI_CLANG) && defined(CONFIG_THINLTO) +/* + * LLVM appends a hash to static function names when ThinLTO and CFI are + * both enabled, which causes confusion and potentially breaks user space + * tools, so we will strip the postfix from expanded symbol names. + */ +static inline char *cleanup_symbol_name(char *s) +{ + char *res = NULL; + + res = strrchr(s, '$'); + if (res) + *res = '\0'; + + return res; +} +#else +static inline char *cleanup_symbol_name(char *s) { return NULL; } +#endif + /* Lookup the address for this symbol. Returns 0 if not found. */ unsigned long kallsyms_lookup_name(const char *name) { @@ -170,6 +190,9 @@ unsigned long kallsyms_lookup_name(const char *name) if (strcmp(namebuf, name) == 0) return kallsyms_sym_address(i); + + if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) + return kallsyms_sym_address(i); } return module_kallsyms_lookup_name(name); } @@ -268,30 +291,6 @@ int kallsyms_lookup_size_offset(unsigned long addr, unsigned long *symbolsize, !!__bpf_address_lookup(addr, symbolsize, offset, namebuf); } -#ifdef CONFIG_CFI_CLANG -/* - * LLVM appends .cfi to function names when CONFIG_CFI_CLANG is enabled, - * which causes confusion and potentially breaks user space tools, so we - * will strip the postfix from expanded symbol names. - */ -static inline void cleanup_symbol_name(char *s) -{ - char *res; - -#ifdef CONFIG_THINLTO - /* Filter out hashes from static functions */ - res = strrchr(s, '$'); - if (res) - *res = '\0'; -#endif - res = strrchr(s, '.'); - if (res && !strcmp(res, ".cfi")) - *res = '\0'; -} -#else -static inline void cleanup_symbol_name(char *s) {} -#endif - /* * Lookup an address * - modname is set to NULL if it's in the kernel. diff --git a/kernel/module.c b/kernel/module.c index b8c25c751586..88ff3060bdb6 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -4235,10 +4235,10 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, static void cfi_init(struct module *mod) { #ifdef CONFIG_CFI_CLANG - preempt_disable(); + rcu_read_lock_sched(); mod->cfi_check = (cfi_check_fn)mod_find_symname(mod, CFI_CHECK_FN_NAME); - preempt_enable(); + rcu_read_unlock_sched(); cfi_module_add(mod, module_addr_min, module_addr_max); #endif }