Blame view
mm/balloon_compaction.c
6.77 KB
18468d93e
|
1 2 3 4 5 6 7 8 9 10 11 12 13 |
/* * mm/balloon_compaction.c * * Common interface for making balloon pages movable by compaction. * * Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <aquini@redhat.com> */ #include <linux/mm.h> #include <linux/slab.h> #include <linux/export.h> #include <linux/balloon_compaction.h> /* |
18468d93e
|
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
* balloon_page_enqueue - allocates a new page and inserts it into the balloon * page list. * @b_dev_info: balloon device decriptor where we will insert a new page to * * Driver must call it to properly allocate a new enlisted balloon page * before definetively removing it from the guest system. * This function returns the page address for the recently enqueued page or * NULL in the case we fail to allocate a new page this turn. */ struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info) { unsigned long flags; struct page *page = alloc_page(balloon_mapping_gfp_mask() | __GFP_NOMEMALLOC | __GFP_NORETRY); if (!page) return NULL; /* * Block others from accessing the 'page' when we get around to * establishing additional references. We should be the only one * holding a reference to the 'page' at this point. */ BUG_ON(!trylock_page(page)); spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
9d1ba8056
|
38 |
balloon_page_insert(b_dev_info, page); |
09316c09d
|
39 |
__count_vm_event(BALLOON_INFLATE); |
18468d93e
|
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 |
spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); unlock_page(page); return page; } EXPORT_SYMBOL_GPL(balloon_page_enqueue); /* * balloon_page_dequeue - removes a page from balloon's page list and returns * the its address to allow the driver release the page. * @b_dev_info: balloon device decriptor where we will grab a page from. * * Driver must call it to properly de-allocate a previous enlisted balloon page * before definetively releasing it back to the guest system. * This function returns the page address for the recently dequeued page or * NULL in the case we find balloon's page list temporarily empty due to * compaction isolated pages. */ struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info) { struct page *page, *tmp; unsigned long flags; bool dequeued_page; dequeued_page = false; list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) { /* * Block others from accessing the 'page' while we get around * establishing additional references and preparing the 'page' * to be released by the balloon driver. */ if (trylock_page(page)) { |
4d88e6f7d
|
71 |
#ifdef CONFIG_BALLOON_COMPACTION |
d6d86c0a7
|
72 73 74 75 76 |
if (!PagePrivate(page)) { /* raced with isolation */ unlock_page(page); continue; } |
4d88e6f7d
|
77 |
#endif |
18468d93e
|
78 |
spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
18468d93e
|
79 |
balloon_page_delete(page); |
09316c09d
|
80 |
__count_vm_event(BALLOON_DEFLATE); |
18468d93e
|
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); unlock_page(page); dequeued_page = true; break; } } if (!dequeued_page) { /* * If we are unable to dequeue a balloon page because the page * list is empty and there is no isolated pages, then something * went out of track and some balloon pages are lost. * BUG() here, otherwise the balloon driver may get stuck into * an infinite loop while attempting to release all its pages. */ spin_lock_irqsave(&b_dev_info->pages_lock, flags); if (unlikely(list_empty(&b_dev_info->pages) && !b_dev_info->isolated_pages)) BUG(); spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); page = NULL; } return page; } EXPORT_SYMBOL_GPL(balloon_page_dequeue); #ifdef CONFIG_BALLOON_COMPACTION |
18468d93e
|
108 109 110 |
static inline void __isolate_balloon_page(struct page *page) { |
9d1ba8056
|
111 |
struct balloon_dev_info *b_dev_info = balloon_page_device(page); |
18468d93e
|
112 |
unsigned long flags; |
d6d86c0a7
|
113 |
|
18468d93e
|
114 |
spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
d6d86c0a7
|
115 |
ClearPagePrivate(page); |
18468d93e
|
116 117 118 119 120 121 122 |
list_del(&page->lru); b_dev_info->isolated_pages++; spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); } static inline void __putback_balloon_page(struct page *page) { |
9d1ba8056
|
123 |
struct balloon_dev_info *b_dev_info = balloon_page_device(page); |
18468d93e
|
124 |
unsigned long flags; |
d6d86c0a7
|
125 |
|
18468d93e
|
126 |
spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
d6d86c0a7
|
127 |
SetPagePrivate(page); |
18468d93e
|
128 129 130 131 |
list_add(&page->lru, &b_dev_info->pages); b_dev_info->isolated_pages--; spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); } |
18468d93e
|
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 |
/* __isolate_lru_page() counterpart for a ballooned page */ bool balloon_page_isolate(struct page *page) { /* * Avoid burning cycles with pages that are yet under __free_pages(), * or just got freed under us. * * In case we 'win' a race for a balloon page being freed under us and * raise its refcount preventing __free_pages() from doing its job * the put_page() at the end of this block will take care of * release this page, thus avoiding a nasty leakage. */ if (likely(get_page_unless_zero(page))) { /* * As balloon pages are not isolated from LRU lists, concurrent * compaction threads can race against page migration functions * as well as race against the balloon driver releasing a page. * * In order to avoid having an already isolated balloon page * being (wrongly) re-isolated while it is under migration, * or to avoid attempting to isolate pages being released by * the balloon driver, lets be sure we have the page lock * before proceeding with the balloon page isolation steps. */ if (likely(trylock_page(page))) { /* |
d6d86c0a7
|
158 |
* A ballooned page, by default, has PagePrivate set. |
18468d93e
|
159 |
* Prevent concurrent compaction threads from isolating |
d6d86c0a7
|
160 |
* an already isolated balloon page by clearing it. |
18468d93e
|
161 |
*/ |
d6d86c0a7
|
162 |
if (balloon_page_movable(page)) { |
18468d93e
|
163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 |
__isolate_balloon_page(page); unlock_page(page); return true; } unlock_page(page); } put_page(page); } return false; } /* putback_lru_page() counterpart for a ballooned page */ void balloon_page_putback(struct page *page) { /* * 'lock_page()' stabilizes the page and prevents races against * concurrent isolation threads attempting to re-isolate it. */ lock_page(page); if (__is_movable_balloon_page(page)) { __putback_balloon_page(page); /* drop the extra ref count taken for page isolation */ put_page(page); } else { WARN_ON(1); |
f0b791a34
|
189 |
dump_page(page, "not movable balloon page"); |
18468d93e
|
190 191 192 193 194 195 196 197 |
} unlock_page(page); } /* move_to_new_page() counterpart for a ballooned page */ int balloon_page_migrate(struct page *newpage, struct page *page, enum migrate_mode mode) { |
9d1ba8056
|
198 |
struct balloon_dev_info *balloon = balloon_page_device(page); |
18468d93e
|
199 200 201 202 203 204 205 206 207 208 |
int rc = -EAGAIN; /* * Block others from accessing the 'newpage' when we get around to * establishing additional references. We should be the only one * holding a reference to the 'newpage' at this point. */ BUG_ON(!trylock_page(newpage)); if (WARN_ON(!__is_movable_balloon_page(page))) { |
f0b791a34
|
209 |
dump_page(page, "not movable balloon page"); |
18468d93e
|
210 211 212 |
unlock_page(newpage); return rc; } |
9d1ba8056
|
213 214 |
if (balloon && balloon->migratepage) rc = balloon->migratepage(balloon, newpage, page, mode); |
18468d93e
|
215 216 217 218 219 |
unlock_page(newpage); return rc; } #endif /* CONFIG_BALLOON_COMPACTION */ |