Commit e587cadd authored by Mathieu Desnoyers's avatar Mathieu Desnoyers Committed by Ingo Molnar

x86: enhance DEBUG_RODATA support - alternatives

Fix a memcpy that should be a text_poke (in apply_alternatives).

Use kernel_wp_save/kernel_wp_restore in text_poke to support DEBUG_RODATA
correctly and so the CPU HOTPLUG special case can be removed.

Add text_poke_early, for alternatives and paravirt boot-time and module load
time patching.

Changelog:

- Fix text_set and text_poke alignment check (mixed up bitwise and and or)
- Remove text_set
- Export add_nops, so it can be used by others.
- Document text_poke_early.
- Remove clflush, since it breaks some VIA architectures and is not strictly
  necessary.
- Add kerneldoc to text_poke and text_poke_early.
- Create a second vmap instead of using the WP bit to support Xen and VMI.
- Move local_irq disable within text_poke and text_poke_early to be able to
  be sleepable in these functions.
Signed-off-by: default avatarMathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
CC: Andi Kleen <andi@firstfloor.org>
CC: pageexec@freemail.hu
CC: H. Peter Anvin <hpa@zytor.com>
CC: Jeremy Fitzhardinge <jeremy@goop.org>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 77bf90ed
...@@ -11,6 +11,8 @@ ...@@ -11,6 +11,8 @@
#include <asm/mce.h> #include <asm/mce.h>
#include <asm/nmi.h> #include <asm/nmi.h>
#include <asm/vsyscall.h> #include <asm/vsyscall.h>
#include <asm/cacheflush.h>
#include <asm/io.h>
#define MAX_PATCH_LEN (255-1) #define MAX_PATCH_LEN (255-1)
...@@ -177,7 +179,7 @@ static const unsigned char*const * find_nop_table(void) ...@@ -177,7 +179,7 @@ static const unsigned char*const * find_nop_table(void)
#endif /* CONFIG_X86_64 */ #endif /* CONFIG_X86_64 */
/* Use this to add nops to a buffer, then text_poke the whole buffer. */ /* Use this to add nops to a buffer, then text_poke the whole buffer. */
static void add_nops(void *insns, unsigned int len) void add_nops(void *insns, unsigned int len)
{ {
const unsigned char *const *noptable = find_nop_table(); const unsigned char *const *noptable = find_nop_table();
...@@ -190,6 +192,7 @@ static void add_nops(void *insns, unsigned int len) ...@@ -190,6 +192,7 @@ static void add_nops(void *insns, unsigned int len)
len -= noplen; len -= noplen;
} }
} }
EXPORT_SYMBOL_GPL(add_nops);
extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
extern u8 *__smp_locks[], *__smp_locks_end[]; extern u8 *__smp_locks[], *__smp_locks_end[];
...@@ -223,7 +226,7 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end) ...@@ -223,7 +226,7 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
memcpy(insnbuf, a->replacement, a->replacementlen); memcpy(insnbuf, a->replacement, a->replacementlen);
add_nops(insnbuf + a->replacementlen, add_nops(insnbuf + a->replacementlen,
a->instrlen - a->replacementlen); a->instrlen - a->replacementlen);
text_poke(instr, insnbuf, a->instrlen); text_poke_early(instr, insnbuf, a->instrlen);
} }
} }
...@@ -284,7 +287,6 @@ void alternatives_smp_module_add(struct module *mod, char *name, ...@@ -284,7 +287,6 @@ void alternatives_smp_module_add(struct module *mod, char *name,
void *text, void *text_end) void *text, void *text_end)
{ {
struct smp_alt_module *smp; struct smp_alt_module *smp;
unsigned long flags;
if (noreplace_smp) if (noreplace_smp)
return; return;
...@@ -310,39 +312,37 @@ void alternatives_smp_module_add(struct module *mod, char *name, ...@@ -310,39 +312,37 @@ void alternatives_smp_module_add(struct module *mod, char *name,
__func__, smp->locks, smp->locks_end, __func__, smp->locks, smp->locks_end,
smp->text, smp->text_end, smp->name); smp->text, smp->text_end, smp->name);
spin_lock_irqsave(&smp_alt, flags); spin_lock(&smp_alt);
list_add_tail(&smp->next, &smp_alt_modules); list_add_tail(&smp->next, &smp_alt_modules);
if (boot_cpu_has(X86_FEATURE_UP)) if (boot_cpu_has(X86_FEATURE_UP))
alternatives_smp_unlock(smp->locks, smp->locks_end, alternatives_smp_unlock(smp->locks, smp->locks_end,
smp->text, smp->text_end); smp->text, smp->text_end);
spin_unlock_irqrestore(&smp_alt, flags); spin_unlock(&smp_alt);
} }
void alternatives_smp_module_del(struct module *mod) void alternatives_smp_module_del(struct module *mod)
{ {
struct smp_alt_module *item; struct smp_alt_module *item;
unsigned long flags;
if (smp_alt_once || noreplace_smp) if (smp_alt_once || noreplace_smp)
return; return;
spin_lock_irqsave(&smp_alt, flags); spin_lock(&smp_alt);
list_for_each_entry(item, &smp_alt_modules, next) { list_for_each_entry(item, &smp_alt_modules, next) {
if (mod != item->mod) if (mod != item->mod)
continue; continue;
list_del(&item->next); list_del(&item->next);
spin_unlock_irqrestore(&smp_alt, flags); spin_unlock(&smp_alt);
DPRINTK("%s: %s\n", __func__, item->name); DPRINTK("%s: %s\n", __func__, item->name);
kfree(item); kfree(item);
return; return;
} }
spin_unlock_irqrestore(&smp_alt, flags); spin_unlock(&smp_alt);
} }
void alternatives_smp_switch(int smp) void alternatives_smp_switch(int smp)
{ {
struct smp_alt_module *mod; struct smp_alt_module *mod;
unsigned long flags;
#ifdef CONFIG_LOCKDEP #ifdef CONFIG_LOCKDEP
/* /*
...@@ -359,7 +359,7 @@ void alternatives_smp_switch(int smp) ...@@ -359,7 +359,7 @@ void alternatives_smp_switch(int smp)
return; return;
BUG_ON(!smp && (num_online_cpus() > 1)); BUG_ON(!smp && (num_online_cpus() > 1));
spin_lock_irqsave(&smp_alt, flags); spin_lock(&smp_alt);
/* /*
* Avoid unnecessary switches because it forces JIT based VMs to * Avoid unnecessary switches because it forces JIT based VMs to
...@@ -383,7 +383,7 @@ void alternatives_smp_switch(int smp) ...@@ -383,7 +383,7 @@ void alternatives_smp_switch(int smp)
mod->text, mod->text_end); mod->text, mod->text_end);
} }
smp_mode = smp; smp_mode = smp;
spin_unlock_irqrestore(&smp_alt, flags); spin_unlock(&smp_alt);
} }
#endif #endif
...@@ -411,7 +411,7 @@ void apply_paravirt(struct paravirt_patch_site *start, ...@@ -411,7 +411,7 @@ void apply_paravirt(struct paravirt_patch_site *start,
/* Pad the rest with nops */ /* Pad the rest with nops */
add_nops(insnbuf + used, p->len - used); add_nops(insnbuf + used, p->len - used);
text_poke(p->instr, insnbuf, p->len); text_poke_early(p->instr, insnbuf, p->len);
} }
} }
extern struct paravirt_patch_site __start_parainstructions[], extern struct paravirt_patch_site __start_parainstructions[],
...@@ -420,8 +420,6 @@ extern struct paravirt_patch_site __start_parainstructions[], ...@@ -420,8 +420,6 @@ extern struct paravirt_patch_site __start_parainstructions[],
void __init alternative_instructions(void) void __init alternative_instructions(void)
{ {
unsigned long flags;
/* The patching is not fully atomic, so try to avoid local interruptions /* The patching is not fully atomic, so try to avoid local interruptions
that might execute the to be patched code. that might execute the to be patched code.
Other CPUs are not running. */ Other CPUs are not running. */
...@@ -430,7 +428,6 @@ void __init alternative_instructions(void) ...@@ -430,7 +428,6 @@ void __init alternative_instructions(void)
stop_mce(); stop_mce();
#endif #endif
local_irq_save(flags);
apply_alternatives(__alt_instructions, __alt_instructions_end); apply_alternatives(__alt_instructions, __alt_instructions_end);
/* switch to patch-once-at-boottime-only mode and free the /* switch to patch-once-at-boottime-only mode and free the
...@@ -462,7 +459,6 @@ void __init alternative_instructions(void) ...@@ -462,7 +459,6 @@ void __init alternative_instructions(void)
} }
#endif #endif
apply_paravirt(__parainstructions, __parainstructions_end); apply_paravirt(__parainstructions, __parainstructions_end);
local_irq_restore(flags);
if (smp_alt_once) if (smp_alt_once)
free_init_pages("SMP alternatives", free_init_pages("SMP alternatives",
...@@ -475,18 +471,64 @@ void __init alternative_instructions(void) ...@@ -475,18 +471,64 @@ void __init alternative_instructions(void)
#endif #endif
} }
/* /**
* Warning: * text_poke_early - Update instructions on a live kernel at boot time
* @addr: address to modify
* @opcode: source of the copy
* @len: length to copy
*
* When you use this code to patch more than one byte of an instruction * When you use this code to patch more than one byte of an instruction
* you need to make sure that other CPUs cannot execute this code in parallel. * you need to make sure that other CPUs cannot execute this code in parallel.
* Also no thread must be currently preempted in the middle of these instructions. * Also no thread must be currently preempted in the middle of these
* And on the local CPU you need to be protected again NMI or MCE handlers * instructions. And on the local CPU you need to be protected again NMI or MCE
* seeing an inconsistent instruction while you patch. * handlers seeing an inconsistent instruction while you patch.
*/ */
void __kprobes text_poke(void *addr, unsigned char *opcode, int len) void *text_poke_early(void *addr, const void *opcode, size_t len)
{ {
unsigned long flags;
local_irq_save(flags);
memcpy(addr, opcode, len); memcpy(addr, opcode, len);
local_irq_restore(flags);
sync_core();
/* Could also do a CLFLUSH here to speed up CPU recovery; but
that causes hangs on some VIA CPUs. */
return addr;
}
/**
* text_poke - Update instructions on a live kernel
* @addr: address to modify
* @opcode: source of the copy
* @len: length to copy
*
* Only atomic text poke/set should be allowed when not doing early patching.
* It means the size must be writable atomically and the address must be aligned
* in a way that permits an atomic write. It also makes sure we fit on a single
* page.
*/
void *__kprobes text_poke(void *addr, const void *opcode, size_t len)
{
unsigned long flags;
char *vaddr;
int nr_pages = 2;
BUG_ON(len > sizeof(long));
BUG_ON((((long)addr + len - 1) & ~(sizeof(long) - 1))
- ((long)addr & ~(sizeof(long) - 1)));
{
struct page *pages[2] = { virt_to_page(addr),
virt_to_page(addr + PAGE_SIZE) };
if (!pages[1])
nr_pages = 1;
vaddr = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
BUG_ON(!vaddr);
local_irq_save(flags);
memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len);
local_irq_restore(flags);
vunmap(vaddr);
}
sync_core(); sync_core();
/* Could also do a CLFLUSH here to speed up CPU recovery; but /* Could also do a CLFLUSH here to speed up CPU recovery; but
that causes hangs on some VIA CPUs. */ that causes hangs on some VIA CPUs. */
return addr;
} }
...@@ -156,6 +156,27 @@ apply_paravirt(struct paravirt_patch_site *start, ...@@ -156,6 +156,27 @@ apply_paravirt(struct paravirt_patch_site *start,
#define __parainstructions_end NULL #define __parainstructions_end NULL
#endif #endif
extern void text_poke(void *addr, unsigned char *opcode, int len); extern void add_nops(void *insns, unsigned int len);
/*
* Clear and restore the kernel write-protection flag on the local CPU.
* Allows the kernel to edit read-only pages.
* Side-effect: any interrupt handler running between save and restore will have
* the ability to write to read-only pages.
*
* Warning:
* Code patching in the UP case is safe if NMIs and MCE handlers are stopped and
* no thread can be preempted in the instructions being modified (no iret to an
* invalid instruction possible) or if the instructions are changed from a
* consistent state to another consistent state atomically.
* More care must be taken when modifying code in the SMP case because of
* Intel's errata.
* On the local CPU you need to be protected again NMI or MCE handlers seeing an
* inconsistent instruction while you patch.
* The _early version expects the memory to already be RW.
*/
extern void *text_poke(void *addr, const void *opcode, size_t len);
extern void *text_poke_early(void *addr, const void *opcode, size_t len);
#endif /* _ASM_X86_ALTERNATIVE_H */ #endif /* _ASM_X86_ALTERNATIVE_H */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment