Eric Lee / smarc-ti-linux-kernel

Download as
Browse Code ยป

Commit 587fe586f44a48f9691001ba6c45b86c8e4ba21f

Authored by Mel Gorman
Committed by Ingo Molnar
1 parent 42836f5f8b
Exists in master and in 20 other branches dlt-processor-sdk-linux-03.00.00.04, newt-ti-linux-3.12.y, processor-sdk-linux-01.00.00, processor-sdk-linux-02.00.01, smarc-ti-linux-3.12.10, smarc-ti-linux-3.12.y, smarc-ti-linux-3.14.y, smarc-ti-linux-3.15.y, smarc-ti-lsk-linux-4.1.y, smarct3x-processor-sdk-04.01.00.06, smarct3x-processor-sdk-linux-02.00.01, smarct3x-processor-sdk-linux-03.00.00.04, smarct4x-800-processor-sdk-linux-02.00.01, smarct4x-processor-sdk-04.01.00.06, smarct4x-processor-sdk-linux-02.00.01, smarct4x-processor-sdk-linux-03.00.00.04, ti-linux-3.12.y, ti-linux-3.14.y, ti-linux-3.15.y, ti-lsk-linux-4.1.y

mm: Prevent parallel splits during THP migration 

THP migrations are serialised by the page lock but on its own that does
not prevent THP splits. If the page is split during THP migration then
the pmd_same checks will prevent page table corruption but the unlock page
and other fix-ups potentially will cause corruption. This patch takes the
anon_vma lock to prevent parallel splits during migration.

Signed-off-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Cc: <stable@kernel.org>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1381141781-10992-7-git-send-email-mgorman@suse.de
Signed-off-by: Ingo Molnar <mingo@kernel.org>

Showing 1 changed file with 30 additions and 14 deletions Side-by-side Diff

... ... @@ -1278,18 +1278,18 @@
1278 1278 int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
1279 1279 unsigned long addr, pmd_t pmd, pmd_t *pmdp)
1280 1280 {
  1281 + struct anon_vma *anon_vma = NULL;
1281 1282 struct page *page;
1282 1283 unsigned long haddr = addr & HPAGE_PMD_MASK;
1283 1284 int target_nid;
1284 1285 int current_nid = -1;
1285   - bool migrated;
  1286 + bool migrated, page_locked;
1286 1287  
1287 1288 spin_lock(&mm->page_table_lock);
1288 1289 if (unlikely(!pmd_same(pmd, *pmdp)))
1289 1290 goto out_unlock;
1290 1291  
1291 1292 page = pmd_page(pmd);
1292   - get_page(page);
1293 1293 current_nid = page_to_nid(page);
1294 1294 count_vm_numa_event(NUMA_HINT_FAULTS);
1295 1295 if (current_nid == numa_node_id())
1296 1296  
1297 1297  
... ... @@ -1299,12 +1299,29 @@
1299 1299 * Acquire the page lock to serialise THP migrations but avoid dropping
1300 1300 * page_table_lock if at all possible
1301 1301 */
1302   - if (trylock_page(page))
1303   - goto got_lock;
  1302 + page_locked = trylock_page(page);
  1303 + target_nid = mpol_misplaced(page, vma, haddr);
  1304 + if (target_nid == -1) {
  1305 + /* If the page was locked, there are no parallel migrations */
  1306 + if (page_locked) {
  1307 + unlock_page(page);
  1308 + goto clear_pmdnuma;
  1309 + }
1304 1310  
1305   - /* Serialise against migrationa and check placement check placement */
  1311 + /* Otherwise wait for potential migrations and retry fault */
  1312 + spin_unlock(&mm->page_table_lock);
  1313 + wait_on_page_locked(page);
  1314 + goto out;
  1315 + }
  1316 +
  1317 + /* Page is misplaced, serialise migrations and parallel THP splits */
  1318 + get_page(page);
1306 1319 spin_unlock(&mm->page_table_lock);
1307   - lock_page(page);
  1320 + if (!page_locked) {
  1321 + lock_page(page);
  1322 + page_locked = true;
  1323 + }
  1324 + anon_vma = page_lock_anon_vma_read(page);
1308 1325  
1309 1326 /* Confirm the PTE did not while locked */
1310 1327 spin_lock(&mm->page_table_lock);
... ... @@ -1314,14 +1331,6 @@
1314 1331 goto out_unlock;
1315 1332 }
1316 1333  
1317   -got_lock:
1318   - target_nid = mpol_misplaced(page, vma, haddr);
1319   - if (target_nid == -1) {
1320   - unlock_page(page);
1321   - put_page(page);
1322   - goto clear_pmdnuma;
1323   - }
1324   -
1325 1334 /* Migrate the THP to the requested node */
1326 1335 spin_unlock(&mm->page_table_lock);
1327 1336 migrated = migrate_misplaced_transhuge_page(mm, vma,
... ... @@ -1330,6 +1339,8 @@
1330 1339 goto check_same;
1331 1340  
1332 1341 task_numa_fault(target_nid, HPAGE_PMD_NR, true);
  1342 + if (anon_vma)
  1343 + page_unlock_anon_vma_read(anon_vma);
1333 1344 return 0;
1334 1345  
1335 1346 check_same:
... ... @@ -1346,6 +1357,11 @@
1346 1357 update_mmu_cache_pmd(vma, addr, pmdp);
1347 1358 out_unlock:
1348 1359 spin_unlock(&mm->page_table_lock);
  1360 +
  1361 +out:
  1362 + if (anon_vma)
  1363 + page_unlock_anon_vma_read(anon_vma);
  1364 +
1349 1365 if (current_nid != -1)
1350 1366 task_numa_fault(current_nid, HPAGE_PMD_NR, false);
1351 1367 return 0;