Commit 546e0d27 authored by Andrew Morton's avatar Andrew Morton Committed by Linus Torvalds

[PATCH] swsusp: read speedup

Implement async reads for swsusp resuming.

Crufty old PIII testbox:
	15.7 MB/s -> 20.3 MB/s

Sony Vaio:
	14.6 MB/s -> 33.3 MB/s

I didn't implement the post-resume bio_set_pages_dirty().  I don't really
understand why resume needs to run set_page_dirty() against these pages.

It might be a worry that this code modifies PG_Uptodate, PG_Error and
PG_Locked against the image pages.  Can this possibly affect the resumed-into
kernel?  Hopefully not, if we're atomically restoring its mem_map?

Cc: Pavel Machek <pavel@ucw.cz>
Cc: "Rafael J. Wysocki" <rjw@sisk.pl>
Cc: Jens Axboe <axboe@suse.de>
Cc: Laurent Riffard <laurent.riffard@free.fr>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 8c002494
...@@ -220,6 +220,7 @@ extern int swap_readpage(struct file *, struct page *); ...@@ -220,6 +220,7 @@ extern int swap_readpage(struct file *, struct page *);
extern int swap_writepage(struct page *page, struct writeback_control *wbc); extern int swap_writepage(struct page *page, struct writeback_control *wbc);
extern int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page, extern int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page,
struct bio **bio_chain); struct bio **bio_chain);
extern int end_swap_bio_read(struct bio *bio, unsigned int bytes_done, int err);
/* linux/mm/swap_state.c */ /* linux/mm/swap_state.c */
extern struct address_space swapper_space; extern struct address_space swapper_space;
......
...@@ -58,6 +58,7 @@ struct snapshot_handle { ...@@ -58,6 +58,7 @@ struct snapshot_handle {
struct pbe *pbe, *last_pbe; struct pbe *pbe, *last_pbe;
void *buffer; void *buffer;
unsigned int buf_offset; unsigned int buf_offset;
int sync_read;
}; };
#define data_of(handle) ((handle).buffer + (handle).buf_offset) #define data_of(handle) ((handle).buffer + (handle).buf_offset)
......
...@@ -314,7 +314,7 @@ static unsigned int unsafe_pages; ...@@ -314,7 +314,7 @@ static unsigned int unsafe_pages;
* and we count them using unsafe_pages * and we count them using unsafe_pages
*/ */
static inline void *alloc_image_page(gfp_t gfp_mask, int safe_needed) static void *alloc_image_page(gfp_t gfp_mask, int safe_needed)
{ {
void *res; void *res;
...@@ -828,13 +828,16 @@ int snapshot_write_next(struct snapshot_handle *handle, size_t count) ...@@ -828,13 +828,16 @@ int snapshot_write_next(struct snapshot_handle *handle, size_t count)
} }
if (!handle->offset) if (!handle->offset)
handle->buffer = buffer; handle->buffer = buffer;
handle->sync_read = 1;
if (handle->prev < handle->page) { if (handle->prev < handle->page) {
if (!handle->prev) { if (!handle->prev) {
error = load_header(handle, (struct swsusp_info *)buffer); error = load_header(handle,
(struct swsusp_info *)buffer);
if (error) if (error)
return error; return error;
} else if (handle->prev <= nr_meta_pages) { } else if (handle->prev <= nr_meta_pages) {
handle->pbe = unpack_orig_addresses(buffer, handle->pbe); handle->pbe = unpack_orig_addresses(buffer,
handle->pbe);
if (!handle->pbe) { if (!handle->pbe) {
error = prepare_image(handle); error = prepare_image(handle);
if (error) if (error)
...@@ -842,10 +845,12 @@ int snapshot_write_next(struct snapshot_handle *handle, size_t count) ...@@ -842,10 +845,12 @@ int snapshot_write_next(struct snapshot_handle *handle, size_t count)
handle->pbe = pagedir_nosave; handle->pbe = pagedir_nosave;
handle->last_pbe = NULL; handle->last_pbe = NULL;
handle->buffer = get_buffer(handle); handle->buffer = get_buffer(handle);
handle->sync_read = 0;
} }
} else { } else {
handle->pbe = handle->pbe->next; handle->pbe = handle->pbe->next;
handle->buffer = get_buffer(handle); handle->buffer = get_buffer(handle);
handle->sync_read = 0;
} }
handle->prev = handle->page; handle->prev = handle->page;
} }
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include <linux/device.h> #include <linux/device.h>
#include <linux/buffer_head.h> #include <linux/buffer_head.h>
#include <linux/bio.h> #include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/swap.h> #include <linux/swap.h>
#include <linux/swapops.h> #include <linux/swapops.h>
#include <linux/pm.h> #include <linux/pm.h>
...@@ -214,6 +215,8 @@ static int wait_on_bio_chain(struct bio **bio_chain) ...@@ -214,6 +215,8 @@ static int wait_on_bio_chain(struct bio **bio_chain)
return 0; return 0;
bio = *bio_chain; bio = *bio_chain;
if (bio == NULL)
return 0;
while (bio) { while (bio) {
struct page *page; struct page *page;
...@@ -349,7 +352,8 @@ int swsusp_write(void) ...@@ -349,7 +352,8 @@ int swsusp_write(void)
int error; int error;
if ((error = swsusp_swap_check())) { if ((error = swsusp_swap_check())) {
printk(KERN_ERR "swsusp: Cannot find swap device, try swapon -a.\n"); printk(KERN_ERR "swsusp: Cannot find swap device, try "
"swapon -a.\n");
return error; return error;
} }
memset(&snapshot, 0, sizeof(struct snapshot_handle)); memset(&snapshot, 0, sizeof(struct snapshot_handle));
...@@ -381,27 +385,6 @@ int swsusp_write(void) ...@@ -381,27 +385,6 @@ int swsusp_write(void)
return error; return error;
} }
/*
* Using bio to read from swap.
* This code requires a bit more work than just using buffer heads
* but, it is the recommended way for 2.5/2.6.
* The following are to signal the beginning and end of I/O. Bios
* finish asynchronously, while we want them to happen synchronously.
* A simple atomic_t, and a wait loop take care of this problem.
*/
static atomic_t io_done = ATOMIC_INIT(0);
static int end_io(struct bio *bio, unsigned int num, int err)
{
if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
printk(KERN_ERR "I/O error reading swsusp image.\n");
return -EIO;
}
atomic_set(&io_done, 0);
return 0;
}
static struct block_device *resume_bdev; static struct block_device *resume_bdev;
/** /**
...@@ -409,15 +392,15 @@ static struct block_device *resume_bdev; ...@@ -409,15 +392,15 @@ static struct block_device *resume_bdev;
* @rw: READ or WRITE. * @rw: READ or WRITE.
* @off physical offset of page. * @off physical offset of page.
* @page: page we're reading or writing. * @page: page we're reading or writing.
* @bio_chain: list of pending biod (for async reading)
* *
* Straight from the textbook - allocate and initialize the bio. * Straight from the textbook - allocate and initialize the bio.
* If we're writing, make sure the page is marked as dirty. * If we're reading, make sure the page is marked as dirty.
* Then submit it and wait. * Then submit it and, if @bio_chain == NULL, wait.
*/ */
static int submit(int rw, pgoff_t page_off, struct page *page,
static int submit(int rw, pgoff_t page_off, void *page) struct bio **bio_chain)
{ {
int error = 0;
struct bio *bio; struct bio *bio;
bio = bio_alloc(GFP_ATOMIC, 1); bio = bio_alloc(GFP_ATOMIC, 1);
...@@ -425,33 +408,40 @@ static int submit(int rw, pgoff_t page_off, void *page) ...@@ -425,33 +408,40 @@ static int submit(int rw, pgoff_t page_off, void *page)
return -ENOMEM; return -ENOMEM;
bio->bi_sector = page_off * (PAGE_SIZE >> 9); bio->bi_sector = page_off * (PAGE_SIZE >> 9);
bio->bi_bdev = resume_bdev; bio->bi_bdev = resume_bdev;
bio->bi_end_io = end_io; bio->bi_end_io = end_swap_bio_read;
if (bio_add_page(bio, virt_to_page(page), PAGE_SIZE, 0) < PAGE_SIZE) { if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
printk("swsusp: ERROR: adding page to bio at %ld\n",page_off); printk("swsusp: ERROR: adding page to bio at %ld\n", page_off);
error = -EFAULT; bio_put(bio);
goto Done; return -EFAULT;
} }
atomic_set(&io_done, 1); lock_page(page);
bio_get(bio);
if (bio_chain == NULL) {
submit_bio(rw | (1 << BIO_RW_SYNC), bio); submit_bio(rw | (1 << BIO_RW_SYNC), bio);
while (atomic_read(&io_done)) wait_on_page_locked(page);
yield();
if (rw == READ) if (rw == READ)
bio_set_pages_dirty(bio); bio_set_pages_dirty(bio);
Done:
bio_put(bio); bio_put(bio);
return error; } else {
get_page(page);
bio->bi_private = *bio_chain;
*bio_chain = bio;
submit_bio(rw | (1 << BIO_RW_SYNC), bio);
}
return 0;
} }
static int bio_read_page(pgoff_t page_off, void *page) static int bio_read_page(pgoff_t page_off, void *addr, struct bio **bio_chain)
{ {
return submit(READ, page_off, page); return submit(READ, page_off, virt_to_page(addr), bio_chain);
} }
static int bio_write_page(pgoff_t page_off, void *page) static int bio_write_page(pgoff_t page_off, void *addr)
{ {
return submit(WRITE, page_off, page); return submit(WRITE, page_off, virt_to_page(addr), NULL);
} }
/** /**
...@@ -476,7 +466,7 @@ static int get_swap_reader(struct swap_map_handle *handle, ...@@ -476,7 +466,7 @@ static int get_swap_reader(struct swap_map_handle *handle,
handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_ATOMIC); handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_ATOMIC);
if (!handle->cur) if (!handle->cur)
return -ENOMEM; return -ENOMEM;
error = bio_read_page(swp_offset(start), handle->cur); error = bio_read_page(swp_offset(start), handle->cur, NULL);
if (error) { if (error) {
release_swap_reader(handle); release_swap_reader(handle);
return error; return error;
...@@ -485,7 +475,8 @@ static int get_swap_reader(struct swap_map_handle *handle, ...@@ -485,7 +475,8 @@ static int get_swap_reader(struct swap_map_handle *handle,
return 0; return 0;
} }
static int swap_read_page(struct swap_map_handle *handle, void *buf) static int swap_read_page(struct swap_map_handle *handle, void *buf,
struct bio **bio_chain)
{ {
unsigned long offset; unsigned long offset;
int error; int error;
...@@ -495,16 +486,17 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf) ...@@ -495,16 +486,17 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf)
offset = handle->cur->entries[handle->k]; offset = handle->cur->entries[handle->k];
if (!offset) if (!offset)
return -EFAULT; return -EFAULT;
error = bio_read_page(offset, buf); error = bio_read_page(offset, buf, bio_chain);
if (error) if (error)
return error; return error;
if (++handle->k >= MAP_PAGE_ENTRIES) { if (++handle->k >= MAP_PAGE_ENTRIES) {
error = wait_on_bio_chain(bio_chain);
handle->k = 0; handle->k = 0;
offset = handle->cur->next_swap; offset = handle->cur->next_swap;
if (!offset) if (!offset)
release_swap_reader(handle); release_swap_reader(handle);
else else if (!error)
error = bio_read_page(offset, handle->cur); error = bio_read_page(offset, handle->cur, NULL);
} }
return error; return error;
} }
...@@ -517,38 +509,48 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf) ...@@ -517,38 +509,48 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf)
static int load_image(struct swap_map_handle *handle, static int load_image(struct swap_map_handle *handle,
struct snapshot_handle *snapshot, struct snapshot_handle *snapshot,
unsigned int nr_pages) unsigned int nr_to_read)
{ {
unsigned int m; unsigned int m;
int ret;
int error = 0; int error = 0;
struct timeval start; struct timeval start;
struct timeval stop; struct timeval stop;
struct bio *bio;
int err2;
unsigned nr_pages;
printk("Loading image data pages (%u pages) ... ", nr_pages); printk("Loading image data pages (%u pages) ... ", nr_to_read);
m = nr_pages / 100; m = nr_to_read / 100;
if (!m) if (!m)
m = 1; m = 1;
nr_pages = 0; nr_pages = 0;
bio = NULL;
do_gettimeofday(&start); do_gettimeofday(&start);
do { for ( ; ; ) {
ret = snapshot_write_next(snapshot, PAGE_SIZE); error = snapshot_write_next(snapshot, PAGE_SIZE);
if (ret > 0) { if (error <= 0)
error = swap_read_page(handle, data_of(*snapshot)); break;
error = swap_read_page(handle, data_of(*snapshot), &bio);
if (error)
break;
if (snapshot->sync_read)
error = wait_on_bio_chain(&bio);
if (error) if (error)
break; break;
if (!(nr_pages % m)) if (!(nr_pages % m))
printk("\b\b\b\b%3d%%", nr_pages / m); printk("\b\b\b\b%3d%%", nr_pages / m);
nr_pages++; nr_pages++;
} }
} while (ret > 0); err2 = wait_on_bio_chain(&bio);
do_gettimeofday(&stop); do_gettimeofday(&stop);
if (!error)
error = err2;
if (!error) { if (!error) {
printk("\b\b\b\bdone\n"); printk("\b\b\b\bdone\n");
if (!snapshot_image_loaded(snapshot)) if (!snapshot_image_loaded(snapshot))
error = -ENODATA; error = -ENODATA;
} }
show_speed(&start, &stop, nr_pages, "Read"); show_speed(&start, &stop, nr_to_read, "Read");
return error; return error;
} }
...@@ -571,7 +573,7 @@ int swsusp_read(void) ...@@ -571,7 +573,7 @@ int swsusp_read(void)
header = (struct swsusp_info *)data_of(snapshot); header = (struct swsusp_info *)data_of(snapshot);
error = get_swap_reader(&handle, swsusp_header.image); error = get_swap_reader(&handle, swsusp_header.image);
if (!error) if (!error)
error = swap_read_page(&handle, header); error = swap_read_page(&handle, header, NULL);
if (!error) if (!error)
error = load_image(&handle, &snapshot, header->pages - 1); error = load_image(&handle, &snapshot, header->pages - 1);
release_swap_reader(&handle); release_swap_reader(&handle);
...@@ -597,7 +599,7 @@ int swsusp_check(void) ...@@ -597,7 +599,7 @@ int swsusp_check(void)
if (!IS_ERR(resume_bdev)) { if (!IS_ERR(resume_bdev)) {
set_blocksize(resume_bdev, PAGE_SIZE); set_blocksize(resume_bdev, PAGE_SIZE);
memset(&swsusp_header, 0, sizeof(swsusp_header)); memset(&swsusp_header, 0, sizeof(swsusp_header));
if ((error = bio_read_page(0, &swsusp_header))) if ((error = bio_read_page(0, &swsusp_header, NULL)))
return error; return error;
if (!memcmp(SWSUSP_SIG, swsusp_header.sig, 10)) { if (!memcmp(SWSUSP_SIG, swsusp_header.sig, 10)) {
memcpy(swsusp_header.sig, swsusp_header.orig_sig, 10); memcpy(swsusp_header.sig, swsusp_header.orig_sig, 10);
......
...@@ -74,7 +74,7 @@ static int end_swap_bio_write(struct bio *bio, unsigned int bytes_done, int err) ...@@ -74,7 +74,7 @@ static int end_swap_bio_write(struct bio *bio, unsigned int bytes_done, int err)
return 0; return 0;
} }
static int end_swap_bio_read(struct bio *bio, unsigned int bytes_done, int err) int end_swap_bio_read(struct bio *bio, unsigned int bytes_done, int err)
{ {
const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
struct page *page = bio->bi_io_vec[0].bv_page; struct page *page = bio->bi_io_vec[0].bv_page;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment