diff options
Diffstat (limited to 'include/linux/jump_label.h')
| -rw-r--r-- | include/linux/jump_label.h | 49 |
1 files changed, 35 insertions, 14 deletions
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index a5079072da6..784304b222b 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -48,6 +48,13 @@ #include <linux/types.h> #include <linux/compiler.h> +#include <linux/bug.h> + +extern bool static_key_initialized; + +#define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \ + "%s used before call to jump_label_init", \ + __func__) #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) @@ -62,6 +69,10 @@ struct static_key { # include <asm/jump_label.h> # define HAVE_JUMP_LABEL +#else +struct static_key { + atomic_t enabled; +}; #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */ enum jump_label_type { @@ -72,20 +83,29 @@ enum jump_label_type { struct module; #include <linux/atomic.h> + +static inline int static_key_count(struct static_key *key) +{ + return atomic_read(&key->enabled); +} + #ifdef HAVE_JUMP_LABEL -#define JUMP_LABEL_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL +#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL +#define JUMP_LABEL_TYPE_MASK 1UL static inline struct jump_entry *jump_label_get_entries(struct static_key *key) { return (struct jump_entry *)((unsigned long)key->entries - & ~JUMP_LABEL_TRUE_BRANCH); + & ~JUMP_LABEL_TYPE_MASK); } static inline bool jump_label_get_branch_default(struct static_key *key) { - if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH) + if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) == + JUMP_LABEL_TYPE_TRUE_BRANCH) return true; return false; } @@ -115,42 +135,43 @@ extern void static_key_slow_inc(struct static_key *key); extern void static_key_slow_dec(struct static_key *key); extern void jump_label_apply_nops(struct module *mod); -#define STATIC_KEY_INIT_TRUE ((struct static_key) \ - { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) -#define STATIC_KEY_INIT_FALSE ((struct static_key) \ - { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) +#define STATIC_KEY_INIT_TRUE ((struct static_key) \ + { .enabled = ATOMIC_INIT(1), \ + .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH }) +#define STATIC_KEY_INIT_FALSE ((struct static_key) \ + { .enabled = ATOMIC_INIT(0), \ + .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH }) #else /* !HAVE_JUMP_LABEL */ -struct static_key { - atomic_t enabled; -}; - static __always_inline void jump_label_init(void) { + static_key_initialized = true; } static __always_inline bool static_key_false(struct static_key *key) { - if (unlikely(atomic_read(&key->enabled)) > 0) + if (unlikely(static_key_count(key) > 0)) return true; return false; } static __always_inline bool static_key_true(struct static_key *key) { - if (likely(atomic_read(&key->enabled)) > 0) + if (likely(static_key_count(key) > 0)) return true; return false; } static inline void static_key_slow_inc(struct static_key *key) { + STATIC_KEY_CHECK_USE(); atomic_inc(&key->enabled); } static inline void static_key_slow_dec(struct static_key *key) { + STATIC_KEY_CHECK_USE(); atomic_dec(&key->enabled); } @@ -179,7 +200,7 @@ static inline int jump_label_apply_nops(struct module *mod) static inline bool static_key_enabled(struct static_key *key) { - return (atomic_read(&key->enabled) > 0); + return static_key_count(key) > 0; } #endif /* _LINUX_JUMP_LABEL_H */ |