Skip to content

L
linux-davinci
Commit 2c750557 authored by Linus Torvalds's avatar Linus Torvalds
Browse files
Options

Merge git://git.kernel.org/pub/scm/linux/kernel/git/rusty/linux-2.6-lguest 

* git://git.kernel.org/pub/scm/linux/kernel/git/rusty/linux-2.6-lguest:
  lguest: documentation update
  lguest: Add to maintainers file.
  lguest: build fix
  lguest: clean up lguest_launcher.h
  lguest: remove unused "wake" element from struct lguest
  lguest: use defines from x86 headers instead of magic numbers
  lguest: example launcher header cleanup.
parents fc42dabe e1e72965
Expand all Show whitespace changes
Inline Side-by-side
Showing with 440 additions and 299 deletions
Documentation/lguest/lguest.c
View file @ 2c750557
This diff is collapsed.
MAINTAINERS
View file @ 2c750557
......@@ -2259,6 +2259,13 @@ L: legousb-devel@lists.sourceforge.net
W: http://legousb.sourceforge.net/
S: Maintained
LGUEST
P: Rusty Russell
M: rusty@rustcorp.com.au
L: lguest@ozlabs.org
W: http://lguest.ozlabs.org/
S: Maintained
LINUX FOR IBM pSERIES (RS/6000)
P: Paul Mackerras
M: paulus@au.ibm.com
......
arch/x86/lguest/boot.c
View file @ 2c750557
......@@ -56,6 +56,7 @@
#include <linux/lguest.h>
#include <linux/lguest_launcher.h>
#include <linux/virtio_console.h>
#include <linux/pm.h>
#include <asm/paravirt.h>
#include <asm/param.h>
#include <asm/page.h>
......@@ -98,7 +99,7 @@ static cycle_t clock_base;
* When lazy_mode is set, it means we're allowed to defer all hypercalls and do
* them as a batch when lazy_mode is eventually turned off. Because hypercalls
* are reasonably expensive, batching them up makes sense. For example, a
* large mmap might update dozens of page table entries: that code calls
* large munmap might update dozens of page table entries: that code calls
* paravirt_enter_lazy_mmu(), does the dozen updates, then calls
* lguest_leave_lazy_mode().
*
......@@ -163,8 +164,8 @@ void async_hcall(unsigned long call,
/*:*/
/*G:033
* Here are our first native-instruction replacements: four functions for
* interrupt control.
* After that diversion we return to our first native-instruction
* replacements: four functions for interrupt control.
*
* The simplest way of implementing these would be to have "turn interrupts
* off" and "turn interrupts on" hypercalls. Unfortunately, this is too slow:
......@@ -183,7 +184,7 @@ static unsigned long save_fl(void)
return lguest_data.irq_enabled;
}
/* "restore_flags" just sets the flags back to the value given. */
/* restore_flags() just sets the flags back to the value given. */
static void restore_fl(unsigned long flags)
{
lguest_data.irq_enabled = flags;
......@@ -356,7 +357,7 @@ static void lguest_cpuid(unsigned int *eax, unsigned int *ebx,
* it. The Host needs to know when the Guest wants to change them, so we have
* a whole series of functions like read_cr0() and write_cr0().
*
* We start with CR0. CR0 allows you to turn on and off all kinds of basic
* We start with cr0. cr0 allows you to turn on and off all kinds of basic
* features, but Linux only really cares about one: the horrifically-named Task
* Switched (TS) bit at bit 3 (ie. 8)
*
......@@ -371,8 +372,7 @@ static void lguest_cpuid(unsigned int *eax, unsigned int *ebx,
static unsigned long current_cr0, current_cr3;
static void lguest_write_cr0(unsigned long val)
{
/* 8 == TS bit. */
lazy_hcall(LHCALL_TS, val & 8, 0, 0);
lazy_hcall(LHCALL_TS, val & X86_CR0_TS, 0, 0);
current_cr0 = val;
}
......@@ -387,10 +387,10 @@ static unsigned long lguest_read_cr0(void)
static void lguest_clts(void)
{
lazy_hcall(LHCALL_TS, 0, 0, 0);
current_cr0 &= ~8U;
current_cr0 &= ~X86_CR0_TS;
}
/* CR2 is the virtual address of the last page fault, which the Guest only ever
/* cr2 is the virtual address of the last page fault, which the Guest only ever
* reads. The Host kindly writes this into our "struct lguest_data", so we
* just read it out of there. */
static unsigned long lguest_read_cr2(void)
......@@ -398,7 +398,7 @@ static unsigned long lguest_read_cr2(void)
return lguest_data.cr2;
}
/* CR3 is the current toplevel pagetable page: the principle is the same as
/* cr3 is the current toplevel pagetable page: the principle is the same as
* cr0. Keep a local copy, and tell the Host when it changes. */
static void lguest_write_cr3(unsigned long cr3)
{
......@@ -411,7 +411,7 @@ static unsigned long lguest_read_cr3(void)
return current_cr3;
}
/* CR4 is used to enable and disable PGE, but we don't care. */
/* cr4 is used to enable and disable PGE, but we don't care. */
static unsigned long lguest_read_cr4(void)
{
return 0;
......@@ -432,7 +432,7 @@ static void lguest_write_cr4(unsigned long val)
* maps virtual addresses to physical addresses using "page tables". We could
* use one huge index of 1 million entries: each address is 4 bytes, so that's
* 1024 pages just to hold the page tables. But since most virtual addresses
* are unused, we use a two level index which saves space. The CR3 register
* are unused, we use a two level index which saves space. The cr3 register
* contains the physical address of the top level "page directory" page, which
* contains physical addresses of up to 1024 second-level pages. Each of these
* second level pages contains up to 1024 physical addresses of actual pages,
......@@ -440,7 +440,7 @@ static void lguest_write_cr4(unsigned long val)
*
* Here's a diagram, where arrows indicate physical addresses:
*
* CR3 ---> +---------+
* cr3 ---> +---------+
* | --------->+---------+
* | | | PADDR1 |
* Top-level | | PADDR2 |
......@@ -498,8 +498,7 @@ static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval)
*
* ... except in early boot when the kernel sets up the initial pagetables,
* which makes booting astonishingly slow. So we don't even tell the Host
* anything changed until we've done the first page table switch.
*/
* anything changed until we've done the first page table switch. */
static void lguest_set_pte(pte_t *ptep, pte_t pteval)
{
*ptep = pteval;
......@@ -720,10 +719,10 @@ static void lguest_time_init(void)
/* Set up the timer interrupt (0) to go to our simple timer routine */
set_irq_handler(0, lguest_time_irq);
/* Our clock structure look like arch/i386/kernel/tsc.c if we can use
* the TSC, otherwise it's a dumb nanosecond-resolution clock. Either
* way, the "rating" is initialized so high that it's always chosen
* over any other clocksource. */
/* Our clock structure looks like arch/x86/kernel/tsc_32.c if we can
* use the TSC, otherwise it's a dumb nanosecond-resolution clock.
* Either way, the "rating" is set so high that it's always chosen over
* any other clocksource. */
if (lguest_data.tsc_khz)
lguest_clock.mult = clocksource_khz2mult(lguest_data.tsc_khz,
lguest_clock.shift);
......@@ -749,7 +748,7 @@ static void lguest_time_init(void)
* to work. They're pretty simple.
*/
/* The Guest needs to tell the host what stack it expects traps to use. For
/* The Guest needs to tell the Host what stack it expects traps to use. For
* native hardware, this is part of the Task State Segment mentioned above in
* lguest_load_tr_desc(), but to help hypervisors there's this special call.
*
......@@ -850,13 +849,16 @@ static __init char *lguest_memory_setup(void)
return "LGUEST";
}
/* Before virtqueues are set up, we use LHCALL_NOTIFY on normal memory to
* produce console output. */
/* We will eventually use the virtio console device to produce console output,
* but before that is set up we use LHCALL_NOTIFY on normal memory to produce
* console output. */
static __init int early_put_chars(u32 vtermno, const char *buf, int count)
{
char scratch[17];
unsigned int len = count;
/* We use a nul-terminated string, so we have to make a copy. Icky,
* huh? */
if (len > sizeof(scratch) - 1)
len = sizeof(scratch) - 1;
scratch[len] = '\0';
......@@ -883,7 +885,7 @@ static __init int early_put_chars(u32 vtermno, const char *buf, int count)
* Our current solution is to allow the paravirt back end to optionally patch
* over the indirect calls to replace them with something more efficient. We
* patch the four most commonly called functions: disable interrupts, enable
* interrupts, restore interrupts and save interrupts. We usually have 10
* interrupts, restore interrupts and save interrupts. We usually have 6 or 10
* bytes to patch into: the Guest versions of these operations are small enough
* that we can fit comfortably.
*
......@@ -1015,7 +1017,7 @@ __init void lguest_init(void)
asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_DS) : "memory");
/* The Host uses the top of the Guest's virtual address space for the
* Host<->Guest Switcher, and it tells us how much it needs in
* Host<->Guest Switcher, and it tells us how big that is in
* lguest_data.reserve_mem, set up on the LGUEST_INIT hypercall. */
reserve_top_address(lguest_data.reserve_mem);
......@@ -1065,6 +1067,6 @@ __init void lguest_init(void)
/*
* This marks the end of stage II of our journey, The Guest.
*
* It is now time for us to explore the nooks and crannies of the three Guest
* devices and complete our understanding of the Guest in "make Drivers".
* It is now time for us to explore the layer of virtual drivers and complete
* our understanding of the Guest in "make Drivers".
*/
arch/x86/lguest/i386_head.S
View file @ 2c750557
......@@ -6,7 +6,7 @@
#include <asm/processor-flags.h>
/*G:020 This is where we begin: head.S notes that the boot header's platform
* type field is "1" (lguest), so calls us here. The boot header is in %esi.
* type field is "1" (lguest), so calls us here.
*
* WARNING: be very careful here! We're running at addresses equal to physical
* addesses (around 0), not above PAGE_OFFSET as most code expectes
......@@ -17,13 +17,15 @@
* boot. */
.section .init.text, "ax", @progbits
ENTRY(lguest_entry)
/* Make initial hypercall now, so we can set up the pagetables. */
/* We make the "initialization" hypercall now to tell the Host about
* us, and also find out where it put our page tables. */
movl $LHCALL_LGUEST_INIT, %eax
movl $lguest_data - __PAGE_OFFSET, %edx
int $LGUEST_TRAP_ENTRY
/* The Host put the toplevel pagetable in lguest_data.pgdir. The movsl
* instruction uses %esi implicitly. */
* instruction uses %esi implicitly as the source for the copy we'
* about to do. */
movl lguest_data - __PAGE_OFFSET + LGUEST_DATA_pgdir, %esi
/* Copy first 32 entries of page directory to __PAGE_OFFSET entries.
......
drivers/lguest/core.c
View file @ 2c750557
......@@ -128,9 +128,12 @@ static void unmap_switcher(void)
__free_pages(switcher_page[i], 0);
}
/*L:305
/*H:032
* Dealing With Guest Memory.
*
* Before we go too much further into the Host, we need to grok the routines
* we use to deal with Guest memory.
*
* When the Guest gives us (what it thinks is) a physical address, we can use
* the normal copy_from_user() & copy_to_user() on the corresponding place in
* the memory region allocated by the Launcher.
......
drivers/lguest/hypercalls.c
View file @ 2c750557
......@@ -90,6 +90,7 @@ static void do_hcall(struct lguest *lg, struct hcall_args *args)
lg->pending_notify = args->arg1;
break;
default:
/* It should be an architecture-specific hypercall. */
if (lguest_arch_do_hcall(lg, args))
kill_guest(lg, "Bad hypercall %li\n", args->arg0);
}
......@@ -157,7 +158,6 @@ static void do_async_hcalls(struct lguest *lg)
* Guest makes a hypercall, we end up here to set things up: */
static void initialize(struct lguest *lg)
{
/* You can't do anything until you're initialized. The Guest knows the
* rules, so we're unforgiving here. */
if (lg->hcall->arg0 != LHCALL_LGUEST_INIT) {
......@@ -174,7 +174,8 @@ static void initialize(struct lguest *lg)
|| get_user(lg->noirq_end, &lg->lguest_data->noirq_end))
kill_guest(lg, "bad guest page %p", lg->lguest_data);
/* We write the current time into the Guest's data page once now. */
/* We write the current time into the Guest's data page once so it can
* set its clock. */
write_timestamp(lg);
/* page_tables.c will also do some setup. */
......@@ -182,8 +183,8 @@ static void initialize(struct lguest *lg)
/* This is the one case where the above accesses might have been the
* first write to a Guest page. This may have caused a copy-on-write
* fault, but the Guest might be referring to the old (read-only)
* page. */
* fault, but the old page might be (read-only) in the Guest
* pagetable. */
guest_pagetable_clear_all(lg);
}
......@@ -220,7 +221,7 @@ void do_hypercalls(struct lguest *lg)
* Normally it doesn't matter: the Guest will run again and
* update the trap number before we come back here.
*
* However, if we are signalled or the Guest sends DMA to the
* However, if we are signalled or the Guest sends I/O to the
* Launcher, the run_guest() loop will exit without running the
* Guest. When it comes back it would try to re-run the
* hypercall. */
......
drivers/lguest/interrupts_and_traps.c
View file @ 2c750557
......@@ -92,8 +92,8 @@ static void set_guest_interrupt(struct lguest *lg, u32 lo, u32 hi, int has_err)
/* Remember that we never let the Guest actually disable interrupts, so
* the "Interrupt Flag" bit is always set. We copy that bit from the
* Guest's "irq_enabled" field into the eflags word: the Guest copies
* it back in "lguest_iret". */
* Guest's "irq_enabled" field into the eflags word: we saw the Guest
* copy it back in "lguest_iret". */
eflags = lg->regs->eflags;
if (get_user(irq_enable, &lg->lguest_data->irq_enabled) == 0
&& !(irq_enable & X86_EFLAGS_IF))
......@@ -124,7 +124,7 @@ static void set_guest_interrupt(struct lguest *lg, u32 lo, u32 hi, int has_err)
kill_guest(lg, "Disabling interrupts");
}
/*H:200
/*H:205
* Virtual Interrupts.
*
* maybe_do_interrupt() gets called before every entry to the Guest, to see if
......@@ -256,19 +256,21 @@ int deliver_trap(struct lguest *lg, unsigned int num)
* bogus one in): if we fail here, the Guest will be killed. */
if (!idt_present(lg->arch.idt[num].a, lg->arch.idt[num].b))
return 0;
set_guest_interrupt(lg, lg->arch.idt[num].a, lg->arch.idt[num].b, has_err(num));
set_guest_interrupt(lg, lg->arch.idt[num].a, lg->arch.idt[num].b,
has_err(num));
return 1;
}
/*H:250 Here's the hard part: returning to the Host every time a trap happens
* and then calling deliver_trap() and re-entering the Guest is slow.
* Particularly because Guest userspace system calls are traps (trap 128).
* Particularly because Guest userspace system calls are traps (usually trap
* 128).
*
* So we'd like to set up the IDT to tell the CPU to deliver traps directly
* into the Guest. This is possible, but the complexities cause the size of
* this file to double! However, 150 lines of code is worth writing for taking
* system calls down from 1750ns to 270ns. Plus, if lguest didn't do it, all
* the other hypervisors would tease it.
* the other hypervisors would beat it up at lunchtime.
*
* This routine indicates if a particular trap number could be delivered
* directly. */
......@@ -331,7 +333,7 @@ void pin_stack_pages(struct lguest *lg)
* change stacks on each context switch. */
void guest_set_stack(struct lguest *lg, u32 seg, u32 esp, unsigned int pages)
{
/* You are not allowd have a stack segment with privilege level 0: bad
/* You are not allowed have a stack segment with privilege level 0: bad
* Guest! */
if ((seg & 0x3) != GUEST_PL)
kill_guest(lg, "bad stack segment %i", seg);
......@@ -350,7 +352,7 @@ void guest_set_stack(struct lguest *lg, u32 seg, u32 esp, unsigned int pages)
* part of the Host: page table handling. */
/*H:235 This is the routine which actually checks the Guest's IDT entry and
* transfers it into our entry in "struct lguest": */
* transfers it into the entry in "struct lguest": */
static void set_trap(struct lguest *lg, struct desc_struct *trap,
unsigned int num, u32 lo, u32 hi)
{
......@@ -456,6 +458,18 @@ void copy_traps(const struct lguest *lg, struct desc_struct *idt,
}
}
/*H:200
* The Guest Clock.
*
* There are two sources of virtual interrupts. We saw one in lguest_user.c:
* the Launcher sending interrupts for virtual devices. The other is the Guest
* timer interrupt.
*
* The Guest uses the LHCALL_SET_CLOCKEVENT hypercall to tell us how long to
* the next timer interrupt (in nanoseconds). We use the high-resolution timer
* infrastructure to set a callback at that time.
*
* 0 means "turn off the clock". */
void guest_set_clockevent(struct lguest *lg, unsigned long delta)
{
ktime_t expires;
......@@ -466,20 +480,27 @@ void guest_set_clockevent(struct lguest *lg, unsigned long delta)
return;
}
/* We use wallclock time here, so the Guest might not be running for
* all the time between now and the timer interrupt it asked for. This
* is almost always the right thing to do. */
expires = ktime_add_ns(ktime_get_real(), delta);
hrtimer_start(&lg->hrt, expires, HRTIMER_MODE_ABS);
}
/* This is the function called when the Guest's timer expires. */
static enum hrtimer_restart clockdev_fn(struct hrtimer *timer)
{
struct lguest *lg = container_of(timer, struct lguest, hrt);
/* Remember the first interrupt is the timer interrupt. */
set_bit(0, lg->irqs_pending);
/* If the Guest is actually stopped, we need to wake it up. */
if (lg->halted)
wake_up_process(lg->tsk);
return HRTIMER_NORESTART;
}
/* This sets up the timer for this Guest. */
void init_clockdev(struct lguest *lg)
{
hrtimer_init(&lg->hrt, CLOCK_REALTIME, HRTIMER_MODE_ABS);
......
drivers/lguest/lg.h
View file @ 2c750557
......@@ -74,9 +74,6 @@ struct lguest
u32 pgdidx;
struct pgdir pgdirs[4];
/* Cached wakeup: we hold a reference to this task. */
struct task_struct *wake;
unsigned long noirq_start, noirq_end;
unsigned long pending_notify; /* pfn from LHCALL_NOTIFY */
......@@ -103,7 +100,7 @@ int lguest_address_ok(const struct lguest *lg,
void __lgread(struct lguest *, void *, unsigned long, unsigned);
void __lgwrite(struct lguest *, unsigned long, const void *, unsigned);
/*L:306 Using memory-copy operations like that is usually inconvient, so we
/*H:035 Using memory-copy operations like that is usually inconvient, so we
* have the following helper macros which read and write a specific type (often
* an unsigned long).
*
......@@ -191,7 +188,7 @@ void write_timestamp(struct lguest *lg);
* Let's step aside for the moment, to study one important routine that's used
* widely in the Host code.
*
* There are many cases where the Guest does something invalid, like pass crap
* There are many cases where the Guest can do something invalid, like pass crap
* to a hypercall. Since only the Guest kernel can make hypercalls, it's quite
* acceptable to simply terminate the Guest and give the Launcher a nicely
* formatted reason. It's also simpler for the Guest itself, which doesn't
......
drivers/lguest/lguest_device.c
View file @ 2c750557
......@@ -53,7 +53,8 @@ struct lguest_device {
* Device configurations
*
* The configuration information for a device consists of a series of fields.
* The device will look for these fields during setup.
* We don't really care what they are: the Launcher set them up, and the driver
* will look at them during setup.
*
* For us these fields come immediately after that device's descriptor in the
* lguest_devices page.
......@@ -122,8 +123,8 @@ static void lg_set_status(struct virtio_device *vdev, u8 status)
* The other piece of infrastructure virtio needs is a "virtqueue": a way of
* the Guest device registering buffers for the other side to read from or
* write into (ie. send and receive buffers). Each device can have multiple
* virtqueues: for example the console has one queue for sending and one for
* receiving.
* virtqueues: for example the console driver uses one queue for sending and
* another for receiving.
*
* Fortunately for us, a very fast shared-memory-plus-descriptors virtqueue
* already exists in virtio_ring.c. We just need to connect it up.
......@@ -158,7 +159,7 @@ static void lg_notify(struct virtqueue *vq)
*
* This is kind of an ugly duckling. It'd be nicer to have a standard
* representation of a virtqueue in the configuration space, but it seems that
* everyone wants to do it differently. The KVM guys want the Guest to
* everyone wants to do it differently. The KVM coders want the Guest to
* allocate its own pages and tell the Host where they are, but for lguest it's
* simpler for the Host to simply tell us where the pages are.
*
......@@ -284,6 +285,8 @@ static void add_lguest_device(struct lguest_device_desc *d)
{
struct lguest_device *ldev;
/* Start with zeroed memory; Linux's device layer seems to count on
* it. */
ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
if (!ldev) {
printk(KERN_EMERG "Cannot allocate lguest dev %u\n",
......
drivers/lguest/lguest_user.c
View file @ 2c750557
......@@ -8,20 +8,22 @@
#include <linux/fs.h>
#include "lg.h"
/*L:315 To force the Guest to stop running and return to the Launcher, the
* Waker sets writes LHREQ_BREAK and the value "1" to /dev/lguest. The
* Launcher then writes LHREQ_BREAK and "0" to release the Waker. */
/*L:055 When something happens, the Waker process needs a way to stop the
* kernel running the Guest and return to the Launcher. So the Waker writes
* LHREQ_BREAK and the value "1" to /dev/lguest to do this. Once the Launcher
* has done whatever needs attention, it writes LHREQ_BREAK and "0" to release
* the Waker. */
static int break_guest_out(struct lguest *lg, const unsigned long __user *input)
{
unsigned long on;
/* Fetch whether they're turning break on or off.. */
/* Fetch whether they're turning break on or off. */
if (get_user(on, input) != 0)
return -EFAULT;
if (on) {
lg->break_out = 1;
/* Pop it out (may be running on different CPU) */
/* Pop it out of the Guest (may be running on different CPU) */
wake_up_process(lg->tsk);
/* Wait for them to reset it */
return wait_event_interruptible(lg->break_wq, !lg->break_out);
......@@ -58,7 +60,7 @@ static ssize_t read(struct file *file, char __user *user, size_t size,loff_t*o)
if (!lg)
return -EINVAL;
/* If you're not the task which owns the guest, go away. */
/* If you're not the task which owns the Guest, go away. */
if (current != lg->tsk)
return -EPERM;
......@@ -92,8 +94,8 @@ static ssize_t read(struct file *file, char __user *user, size_t size,loff_t*o)
* base: The start of the Guest-physical memory inside the Launcher memory.
*
* pfnlimit: The highest (Guest-physical) page number the Guest should be
* allowed to access. The Launcher has to live in Guest memory, so it sets
* this to ensure the Guest can't reach it.
* allowed to access. The Guest memory lives inside the Launcher, so it sets
* this to ensure the Guest can only reach its own memory.
*
* pgdir: The (Guest-physical) address of the top of the initial Guest
* pagetables (which are set up by the Launcher).
......@@ -189,7 +191,7 @@ unlock:
}
/*L:010 The first operation the Launcher does must be a write. All writes
* start with a 32 bit number: for the first write this must be
* start with an unsigned long number: for the first write this must be
* LHREQ_INITIALIZE to set up the Guest. After that the Launcher can use
* writes of other values to send interrupts. */
static ssize_t write(struct file *file, const char __user *in,
......@@ -275,8 +277,7 @@ static int close(struct inode *inode, struct file *file)
* The Launcher is the Host userspace program which sets up, runs and services
* the Guest. In fact, many comments in the Drivers which refer to "the Host"
* doing things are inaccurate: the Launcher does all the device handling for
* the Guest. The Guest can't tell what's done by the the Launcher and what by
* the Host.
* the Guest, but the Guest can't know that.
*
* Just to confuse you: to the Host kernel, the Launcher *is* the Guest and we
* shall see more of that later.
......
drivers/lguest/page_tables.c
View file @ 2c750557
This diff is collapsed.
drivers/lguest/segments.c
View file @ 2c750557
......@@ -12,8 +12,6 @@
#include "lg.h"
/*H:600
* We've almost completed the Host; there's just one file to go!
*
* Segments & The Global Descriptor Table
*
* (That title sounds like a bad Nerdcore group. Not to suggest that there are
......@@ -55,7 +53,7 @@ static int ignored_gdt(unsigned int num)
|| num == GDT_ENTRY_DOUBLEFAULT_TSS);
}
/*H:610 Once the GDT has been changed, we fix the new entries up a little. We
/*H:630 Once the Guest gave us new GDT entries, we fix them up a little. We
* don't care if they're invalid: the worst that can happen is a General
* Protection Fault in the Switcher when it restores a Guest segment register
* which tries to use that entry. Then we kill the Guest for causing such a
......@@ -84,25 +82,33 @@ static void fixup_gdt_table(struct lguest *lg, unsigned start, unsigned end)
}
}
/* This routine is called at boot or modprobe time for each CPU to set up the
* "constant" GDT entries for Guests running on that CPU. */
/*H:610 Like the IDT, we never simply use the GDT the Guest gives us. We keep
* a GDT for each CPU, and copy across the Guest's entries each time we want to
* run the Guest on that CPU.
*
* This routine is called at boot or modprobe time for each CPU to set up the
* constant GDT entries: the ones which are the same no matter what Guest we're
* running. */
void setup_default_gdt_entries(struct lguest_ro_state *state)
{
struct desc_struct *gdt = state->guest_gdt;
unsigned long tss = (unsigned long)&state->guest_tss;
/* The hypervisor segments are full 0-4G segments, privilege level 0 */
/* The Switcher segments are full 0-4G segments, privilege level 0 */
gdt[GDT_ENTRY_LGUEST_CS] = FULL_EXEC_SEGMENT;
gdt[GDT_ENTRY_LGUEST_DS] = FULL_SEGMENT;
/* The TSS segment refers to the TSS entry for this CPU, so we cannot
* copy it from the Guest. Forgive the magic flags */
/* The TSS segment refers to the TSS entry for this particular CPU.
* Forgive the magic flags: the 0x8900 means the entry is Present, it's
* privilege level 0 Available 386 TSS system segment, and the 0x67
* means Saturn is eclipsed by Mercury in the twelfth house. */
gdt[GDT_ENTRY_TSS].a = 0x00000067 | (tss << 16);
gdt[GDT_ENTRY_TSS].b = 0x00008900 | (tss & 0xFF000000)
| ((tss >> 16) & 0x000000FF);
}
/* This routine is called before the Guest is run for the first time. */
/* This routine sets up the initial Guest GDT for booting. All entries start
* as 0 (unusable). */
void setup_guest_gdt(struct lguest *lg)
{
/* Start with full 0-4G segments... */
......@@ -114,13 +120,8 @@ void setup_guest_gdt(struct lguest *lg)
lg->arch.gdt[GDT_ENTRY_KERNEL_DS].b |= (GUEST_PL << 13);
}
/* Like the IDT, we never simply use the GDT the Guest gives us. We set up the
* GDTs for each CPU, then we copy across the entries each time we want to run
* a different Guest on that CPU. */
/* A partial GDT load, for the three "thead-local storage" entries. Otherwise
* it's just like load_guest_gdt(). So much, in fact, it would probably be
* neater to have a single hypercall to cover both. */
/*H:650 An optimization of copy_gdt(), for just the three "thead-local storage"
* entries. */
void copy_gdt_tls(const struct lguest *lg, struct desc_struct *gdt)
{
unsigned int i;
......@@ -129,7 +130,9 @@ void copy_gdt_tls(const struct lguest *lg, struct desc_struct *gdt)
gdt[i] = lg->arch.gdt[i];
}
/* This is the full version */
/*H:640 When the Guest is run on a different CPU, or the GDT entries have
* changed, copy_gdt() is called to copy the Guest's GDT entries across to this
* CPU's GDT. */
void copy_gdt(const struct lguest *lg, struct desc_struct *gdt)
{
unsigned int i;
......@@ -141,7 +144,8 @@ void copy_gdt(const struct lguest *lg, struct desc_struct *gdt)
gdt[i] = lg->arch.gdt[i];
}
/* This is where the Guest asks us to load a new GDT (LHCALL_LOAD_GDT). */
/*H:620 This is where the Guest asks us to load a new GDT (LHCALL_LOAD_GDT).
* We copy it from the Guest and tweak the entries. */
void load_guest_gdt(struct lguest *lg, unsigned long table, u32 num)
{
/* We assume the Guest has the same number of GDT entries as the
......@@ -157,16 +161,22 @@ void load_guest_gdt(struct lguest *lg, unsigned long table, u32 num)
lg->changed |= CHANGED_GDT;
}
/* This is the fast-track version for just changing the three TLS entries.
* Remember that this happens on every context switch, so it's worth
* optimizing. But wouldn't it be neater to have a single hypercall to cover
* both cases? */
void guest_load_tls(struct lguest *lg, unsigned long gtls)
{
struct desc_struct *tls = &lg->arch.gdt[GDT_ENTRY_TLS_MIN];
__lgread(lg, tls, gtls, sizeof(*tls)*GDT_ENTRY_TLS_ENTRIES);
fixup_gdt_table(lg, GDT_ENTRY_TLS_MIN, GDT_ENTRY_TLS_MAX+1);
/* Note that just the TLS entries have changed. */
lg->changed |= CHANGED_GDT_TLS;
}
/*:*/
/*
/*H:660
* With this, we have finished the Host.
*
* Five of the seven parts of our task are complete. You have made it through
......
drivers/lguest/x86/core.c
View file @ 2c750557
......@@ -63,7 +63,7 @@ static struct lguest_pages *lguest_pages(unsigned int cpu)
static DEFINE_PER_CPU(struct lguest *, last_guest);
/*S:010
* We are getting close to the Switcher.
* We approach the Switcher.
*
* Remember that each CPU has two pages which are visible to the Guest when it
* runs on that CPU. This has to contain the state for that Guest: we copy the
......@@ -134,7 +134,7 @@ static void run_guest_once(struct lguest *lg, struct lguest_pages *pages)
*
* The lcall also pushes the old code segment (KERNEL_CS) onto the
* stack, then the address of this call. This stack layout happens to
* exactly match the stack of an interrupt... */
* exactly match the stack layout created by an interrupt... */
asm volatile("pushf; lcall *lguest_entry"
/* This is how we tell GCC that %eax ("a") and %ebx ("b")
* are changed by this routine. The "=" means output. */
......@@ -151,40 +151,46 @@ static void run_guest_once(struct lguest *lg, struct lguest_pages *pages)
}
/*:*/
/*M:002 There are hooks in the scheduler which we can register to tell when we
* get kicked off the CPU (preempt_notifier_register()). This would allow us
* to lazily disable SYSENTER which would regain some performance, and should
* also simplify copy_in_guest_info(). Note that we'd still need to restore
* things when we exit to Launcher userspace, but that's fairly easy.
*
* The hooks were designed for KVM, but we can also put them to good use. :*/
/*H:040 This is the i386-specific code to setup and run the Guest. Interrupts
* are disabled: we own the CPU. */
void lguest_arch_run_guest(struct lguest *lg)
{
/* Remember the awfully-named TS bit? If the Guest has asked
* to set it we set it now, so we can trap and pass that trap
* to the Guest if it uses the FPU. */
/* Remember the awfully-named TS bit? If the Guest has asked to set it
* we set it now, so we can trap and pass that trap to the Guest if it
* uses the FPU. */
if (lg->ts)
lguest_set_ts();
/* SYSENTER is an optimized way of doing system calls. We
* can't allow it because it always jumps to privilege level 0.
* A normal Guest won't try it because we don't advertise it in
* CPUID, but a malicious Guest (or malicious Guest userspace
* program) could, so we tell the CPU to disable it before
* running the Guest. */
/* SYSENTER is an optimized way of doing system calls. We can't allow
* it because it always jumps to privilege level 0. A normal Guest
* won't try it because we don't advertise it in CPUID, but a malicious
* Guest (or malicious Guest userspace program) could, so we tell the
* CPU to disable it before running the Guest. */
if (boot_cpu_has(X86_FEATURE_SEP))
wrmsr(MSR_IA32_SYSENTER_CS, 0, 0);
/* Now we actually run the Guest. It will pop back out when
* something interesting happens, and we can examine its
* registers to see what it was doing. */
/* Now we actually run the Guest. It will return when something
* interesting happens, and we can examine its registers to see what it
* was doing. */
run_guest_once(lg, lguest_pages(raw_smp_processor_id()));
/* The "regs" pointer contains two extra entries which are not
* really registers: a trap number which says what interrupt or
* trap made the switcher code come back, and an error code
* which some traps set. */
/* Note that the "regs" pointer contains two extra entries which are
* not really registers: a trap number which says what interrupt or
* trap made the switcher code come back, and an error code which some
* traps set. */
/* If the Guest page faulted, then the cr2 register will tell
* us the bad virtual address. We have to grab this now,
* because once we re-enable interrupts an interrupt could
* fault and thus overwrite cr2, or we could even move off to a
* different CPU. */
/* If the Guest page faulted, then the cr2 register will tell us the
* bad virtual address. We have to grab this now, because once we
* re-enable interrupts an interrupt could fault and thus overwrite
* cr2, or we could even move off to a different CPU. */
if (lg->regs->trapnum == 14)
lg->arch.last_pagefault = read_cr2();
/* Similarly, if we took a trap because the Guest used the FPU,
......@@ -197,14 +203,15 @@ void lguest_arch_run_guest(struct lguest *lg)
wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0);
}
/*H:130 Our Guest is usually so well behaved; it never tries to do things it
* isn't allowed to. Unfortunately, Linux's paravirtual infrastructure isn't
* quite complete, because it doesn't contain replacements for the Intel I/O
* instructions. As a result, the Guest sometimes fumbles across one during
* the boot process as it probes for various things which are usually attached
* to a PC.
/*H:130 Now we've examined the hypercall code; our Guest can make requests.
* Our Guest is usually so well behaved; it never tries to do things it isn't
* allowed to, and uses hypercalls instead. Unfortunately, Linux's paravirtual
* infrastructure isn't quite complete, because it doesn't contain replacements
* for the Intel I/O instructions. As a result, the Guest sometimes fumbles
* across one during the boot process as it probes for various things which are
* usually attached to a PC.
*
* When the Guest uses one of these instructions, we get trap #13 (General
* When the Guest uses one of these instructions, we get a trap (General
* Protection Fault) and come here. We see if it's one of those troublesome
* instructions and skip over it. We return true if we did. */
static int emulate_insn(struct lguest *lg)
......@@ -275,43 +282,43 @@ static int emulate_insn(struct lguest *lg)
void lguest_arch_handle_trap(struct lguest *lg)
{
switch (lg->regs->trapnum) {
case 13: /* We've intercepted a GPF. */
case 13: /* We've intercepted a General Protection Fault. */
/* Check if this was one of those annoying IN or OUT
* instructions which we need to emulate. If so, we
* just go back into the Guest after we've done it. */
* instructions which we need to emulate. If so, we just go
* back into the Guest after we've done it. */
if (lg->regs->errcode == 0) {
if (emulate_insn(lg))
return;
}
break;
case 14: /* We've intercepted a page fault. */
/* The Guest accessed a virtual address that wasn't
* mapped. This happens a lot: we don't actually set
* up most of the page tables for the Guest at all when
* we start: as it runs it asks for more and more, and
* we set them up as required. In this case, we don't
* even tell the Guest that the fault happened.
case 14: /* We've intercepted a Page Fault. */
/* The Guest accessed a virtual address that wasn't mapped.
* This happens a lot: we don't actually set up most of the
* page tables for the Guest at all when we start: as it runs
* it asks for more and more, and we set them up as
* required. In this case, we don't even tell the Guest that
* the fault happened.
*
* The errcode tells whether this was a read or a
* write, and whether kernel or userspace code. */
* The errcode tells whether this was a read or a write, and
* whether kernel or userspace code. */
if (demand_page(lg, lg->arch.last_pagefault, lg->regs->errcode))
return;
/* OK, it's really not there (or not OK): the Guest
* needs to know. We write out the cr2 value so it
* knows where the fault occurred.
/* OK, it's really not there (or not OK): the Guest needs to
* know. We write out the cr2 value so it knows where the
* fault occurred.
*
* Note that if the Guest were really messed up, this
* could happen before it's done the INITIALIZE
* hypercall, so lg->lguest_data will be NULL */
* Note that if the Guest were really messed up, this could
* happen before it's done the LHCALL_LGUEST_INIT hypercall, so
* lg->lguest_data could be NULL */
if (lg->lguest_data &&
put_user(lg->arch.last_pagefault, &lg->lguest_data->cr2))
kill_guest(lg, "Writing cr2");
break;
case 7: /* We've intercepted a Device Not Available fault. */
/* If the Guest doesn't want to know, we already
* restored the Floating Point Unit, so we just
* continue without telling it. */
/* If the Guest doesn't want to know, we already restored the
* Floating Point Unit, so we just continue without telling
* it. */
if (!lg->ts)
return;
break;
......@@ -536,9 +543,6 @@ int lguest_arch_init_hypercalls(struct lguest *lg)
return 0;
}
/* Now we've examined the hypercall code; our Guest can make requests. There
* is one other way we can do things for the Guest, as we see in
* emulate_insn(). :*/
/*L:030 lguest_arch_setup_regs()
*
......@@ -562,7 +566,7 @@ void lguest_arch_setup_regs(struct lguest *lg, unsigned long start)
* is supposed to always be "1". Bit 9 (0x200) controls whether
* interrupts are enabled. We always leave interrupts enabled while
* running the Guest. */
regs->eflags = 0x202;
regs->eflags = X86_EFLAGS_IF | 0x2;
/* The "Extended Instruction Pointer" register says where the Guest is
* running. */
......@@ -570,8 +574,8 @@ void lguest_arch_setup_regs(struct lguest *lg, unsigned long start)
/* %esi points to our boot information, at physical address 0, so don't
* touch it. */
/* There are a couple of GDT entries the Guest expects when first
* booting. */
setup_guest_gdt(lg);
}
drivers/lguest/x86/switcher_32.S
View file @ 2c750557
......@@ -6,6 +6,37 @@
* are feeling invigorated and refreshed then the next, more challenging stage
* can be found in "make Guest". :*/
/*M:012 Lguest is meant to be simple: my rule of thumb is that 1% more LOC must
* gain at least 1% more performance. Since neither LOC nor performance can be
* measured beforehand, it generally means implementing a feature then deciding
* if it's worth it. And once it's implemented, who can say no?
*
* This is why I haven't implemented this idea myself. I want to, but I
* haven't. You could, though.
*
* The main place where lguest performance sucks is Guest page faulting. When
* a Guest userspace process hits an unmapped page we switch back to the Host,
* walk the page tables, find it's not mapped, switch back to the Guest page
* fault handler, which calls a hypercall to set the page table entry, then
* finally returns to userspace. That's two round-trips.
*
* If we had a small walker in the Switcher, we could quickly check the Guest
* page table and if the page isn't mapped, immediately reflect the fault back
* into the Guest. This means the Switcher would have to know the top of the
* Guest page table and the page fault handler address.
*
* For simplicity, the Guest should only handle the case where the privilege
* level of the fault is 3 and probably only not present or write faults. It
* should also detect recursive faults, and hand the original fault to the
* Host (which is actually really easy).
*
* Two questions remain. Would the performance gain outweigh the complexity?
* And who would write the verse documenting it? :*/
/*M:011 Lguest64 handles NMI. This gave me NMI envy (until I looked at their
* code). It's worth doing though, since it would let us use oprofile in the
* Host when a Guest is running. :*/
/*S:100
* Welcome to the Switcher itself!
*
......@@ -88,7 +119,7 @@ ENTRY(switch_to_guest)
// All saved and there's now five steps before us:
// Stack, GDT, IDT, TSS
// And last of all the page tables are flipped.
// Then last of all the page tables are flipped.
// Yet beware that our stack pointer must be
// Always valid lest an NMI hits
......@@ -103,25 +134,25 @@ ENTRY(switch_to_guest)
lgdt LGUEST_PAGES_guest_gdt_desc(%eax)
// The Guest's IDT we did partially
// Move to the "struct lguest_pages" as well.
// Copy to "struct lguest_pages" as well.
lidt LGUEST_PAGES_guest_idt_desc(%eax)
// The TSS entry which controls traps
// Must be loaded up with "ltr" now:
// The GDT entry that TSS uses
// Changes type when we load it: damn Intel!
// For after we switch over our page tables
// It (as the rest) will be writable no more.
// (The GDT entry TSS needs
// Changes type when we load it: damn Intel!)
// That entry will be read-only: we'd crash.
movl $(GDT_ENTRY_TSS*8), %edx
ltr %dx
// Look back now, before we take this last step!
// The Host's TSS entry was also marked used;
// Let's clear it again, ere we return.
// Let's clear it again for our return.
// The GDT descriptor of the Host
// Points to the table after two "size" bytes
movl (LGUEST_PAGES_host_gdt_desc+2)(%eax), %edx
// Clear the type field of "used" (byte 5, bit 2)
// Clear "used" from type field (byte 5, bit 2)
andb $0xFD, (GDT_ENTRY_TSS*8 + 5)(%edx)
// Once our page table's switched, the Guest is live!
......@@ -131,7 +162,7 @@ ENTRY(switch_to_guest)
// The page table change did one tricky thing:
// The Guest's register page has been mapped
// Writable onto our %esp (stack) --
// Writable under our %esp (stack) --
// We can simply pop off all Guest regs.
popl %eax
popl %ebx
......@@ -152,16 +183,15 @@ ENTRY(switch_to_guest)
addl $8, %esp
// The last five stack slots hold return address
// And everything needed to change privilege
// Into the Guest privilege level of 1,
// And everything needed to switch privilege
// From Switcher's level 0 to Guest's 1,
// And the stack where the Guest had last left it.
// Interrupts are turned back on: we are Guest.
iret
// There are two paths where we switch to the Host
// We treat two paths to switch back to the Host
// Yet both must save Guest state and restore Host
// So we put the routine in a macro.
// We are on our way home, back to the Host
// Interrupted out of the Guest, we come here.
#define SWITCH_TO_HOST \
/* We save the Guest state: all registers first \
* Laid out just as "struct lguest_regs" defines */ \
......@@ -194,7 +224,7 @@ ENTRY(switch_to_guest)
movl %esp, %eax; \
andl $(~(1 << PAGE_SHIFT - 1)), %eax; \
/* Save our trap number: the switch will obscure it \
* (The Guest regs are not mapped here in the Host) \
* (In the Host the Guest regs are not mapped here) \
* %ebx holds it safe for deliver_to_host */ \
movl LGUEST_PAGES_regs_trapnum(%eax), %ebx; \
/* The Host GDT, IDT and stack! \
......@@ -210,9 +240,9 @@ ENTRY(switch_to_guest)
/* Switch to Host's GDT, IDT. */ \
lgdt LGUEST_PAGES_host_gdt_desc(%eax); \
lidt LGUEST_PAGES_host_idt_desc(%eax); \
/* Restore the Host's stack where it's saved regs lie */ \
/* Restore the Host's stack where its saved regs lie */ \
movl LGUEST_PAGES_host_sp(%eax), %esp; \
/* Last the TSS: our Host is complete */ \
/* Last the TSS: our Host is returned */ \
movl $(GDT_ENTRY_TSS*8), %edx; \
ltr %dx; \
/* Restore now the regs saved right at the first. */ \
......@@ -222,14 +252,15 @@ ENTRY(switch_to_guest)
popl %ds; \
popl %es
// Here's where we come when the Guest has just trapped:
// (Which trap we'll see has been pushed on the stack).
// The first path is trod when the Guest has trapped:
// (Which trap it was has been pushed on the stack).
// We need only switch back, and the Host will decode
// Why we came home, and what needs to be done.
return_to_host:
SWITCH_TO_HOST
iret
// We are lead to the second path like so:
// An interrupt, with some cause external
// Has ajerked us rudely from the Guest's code
// Again we must return home to the Host
......@@ -238,7 +269,7 @@ deliver_to_host:
// But now we must go home via that place
// Where that interrupt was supposed to go
// Had we not been ensconced, running the Guest.
// Here we see the cleverness of our stack:
// Here we see the trickness of run_guest_once():
// The Host stack is formed like an interrupt
// With EIP, CS and EFLAGS layered.
// Interrupt handlers end with "iret"
......@@ -263,7 +294,7 @@ deliver_to_host:
xorw %ax, %ax
orl %eax, %edx
// Now the address of the handler's in %edx
// We call it now: its "iret" takes us home.
// We call it now: its "iret" drops us home.
jmp *%edx
// Every interrupt can come to us here
......
include/asm-x86/lguest_hcall.h
View file @ 2c750557
......@@ -18,12 +18,17 @@
#define LHCALL_LOAD_TLS 16
#define LHCALL_NOTIFY 17
#define LGUEST_TRAP_ENTRY 0x1F
#ifndef __ASSEMBLY__
#include <asm/hw_irq.h>
/*G:031 First, how does our Guest contact the Host to ask for privileged
* operations? There are two ways: the direct way is to make a "hypercall",
* to make requests of the Host Itself.
*
* Our hypercall mechanism uses the highest unused trap code (traps 32 and
* above are used by real hardware interrupts). Seventeen hypercalls are
* above are used by real hardware interrupts). Fifteen hypercalls are
* available: the hypercall number is put in the %eax register, and the
* arguments (when required) are placed in %edx, %ebx and %ecx. If a return
* value makes sense, it's returned in %eax.
......@@ -31,20 +36,15 @@
* Grossly invalid calls result in Sudden Death at the hands of the vengeful
* Host, rather than returning failure. This reflects Winston Churchill's
* definition of a gentleman: "someone who is only rude intentionally". */
#define LGUEST_TRAP_ENTRY 0x1F
#ifndef __ASSEMBLY__
#include <asm/hw_irq.h>
static inline unsigned long
hcall(unsigned long call,
unsigned long arg1, unsigned long arg2, unsigned long arg3)
{
/* "int" is the Intel instruction to trigger a trap. */
asm volatile("int $" __stringify(LGUEST_TRAP_ENTRY)
/* The call is in %eax (aka "a"), and can be replaced */
/* The call in %eax (aka "a") might be overwritten */
: "=a"(call)
/* The other arguments are in %eax, %edx, %ebx & %ecx */
/* The arguments are in %eax, %edx, %ebx & %ecx */
: "a"(call), "d"(arg1), "b"(arg2), "c"(arg3)
/* "memory" means this might write somewhere in memory.
* This isn't true for all calls, but it's safe to tell
......
include/linux/lguest.h
View file @ 2c750557
......@@ -12,8 +12,8 @@
#define LG_CLOCK_MAX_DELTA ULONG_MAX
/*G:032 The second method of communicating with the Host is to via "struct
* lguest_data". The Guest's very first hypercall is to tell the Host where
* this is, and then the Guest and Host both publish information in it. :*/
* lguest_data". Once the Guest's initialization hypercall tells the Host where
* this is, the Guest and Host both publish information in it. :*/
struct lguest_data
{
/* 512 == enabled (same as eflags in normal hardware). The Guest
......
include/linux/lguest_launcher.h
View file @ 2c750557
#ifndef _ASM_LGUEST_USER
#define _ASM_LGUEST_USER
#ifndef _LINUX_LGUEST_LAUNCHER
#define _LINUX_LGUEST_LAUNCHER
/* Everything the "lguest" userspace program needs to know. */
#include <linux/types.h>
/* They can register up to 32 arrays of lguest_dma. */
#define LGUEST_MAX_DMA 32
/* At most we can dma 16 lguest_dma in one op. */
#define LGUEST_MAX_DMA_SECTIONS 16
/* How many devices? Assume each one wants up to two dma arrays per device. */
#define LGUEST_MAX_DEVICES (LGUEST_MAX_DMA/2)
/* Where the Host expects the Guest to SEND_DMA console output to. */
#define LGUEST_CONSOLE_DMA_KEY 0
/*D:010
* Drivers
......@@ -20,7 +10,11 @@
* real devices (think of the damage it could do!) we provide virtual devices.
* We could emulate a PCI bus with various devices on it, but that is a fairly
* complex burden for the Host and suboptimal for the Guest, so we have our own
* "lguest" bus and simple drivers.
* simple lguest bus and we use "virtio" drivers. These drivers need a set of
* routines from us which will actually do the virtual I/O, but they handle all
* the net/block/console stuff themselves. This means that if we want to add
* a new device, we simply need to write a new virtio driver and create support
* for it in the Launcher: this code won't need to change.
*
* Devices are described by a simplified ID, a status byte, and some "config"
* bytes which describe this device's configuration. This is placed by the
......@@ -51,9 +45,9 @@ struct lguest_vqconfig {
/* Write command first word is a request. */
enum lguest_req
{
LHREQ_INITIALIZE, /* + pfnlimit, pgdir, start, pageoffset */
LHREQ_INITIALIZE, /* + base, pfnlimit, pgdir, start */
LHREQ_GETDMA, /* No longer used */
LHREQ_IRQ, /* + irq */
LHREQ_BREAK, /* + on/off flag (on blocks until someone does off) */
};
#endif /* _ASM_LGUEST_USER */
#endif /* _LINUX_LGUEST_LAUNCHER */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment