Re: [PATCH v5 3/4] mm: support large folios swapin as a whole for zRAM-like swapfile
From: Dan Carpenter
Date: Mon Jul 29 2024 - 10:17:11 EST
Hi Barry,
kernel test robot noticed the following build warnings:
url: https://github.com/intel-lab-lkp/linux/commits/Barry-Song/mm-swap-introduce-swapcache_prepare_nr-and-swapcache_clear_nr-for-large-folios-swap-in/20240726-181412
base: https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link: https://lore.kernel.org/r/20240726094618.401593-4-21cnbao%40gmail.com
patch subject: [PATCH v5 3/4] mm: support large folios swapin as a whole for zRAM-like swapfile
config: i386-randconfig-141-20240727 (https://download.01.org/0day-ci/archive/20240727/202407270917.18F5rYPH-lkp@xxxxxxxxx/config)
compiler: clang version 18.1.5 (https://github.com/llvm/llvm-project 617a15a9eac96088ae5e9134248d8236e34b91b1)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@xxxxxxxxx>
| Reported-by: Dan Carpenter <dan.carpenter@xxxxxxxxxx>
| Closes: https://lore.kernel.org/r/202407270917.18F5rYPH-lkp@xxxxxxxxx/
smatch warnings:
mm/memory.c:4467 do_swap_page() error: uninitialized symbol 'nr_pages'.
vim +/nr_pages +4467 mm/memory.c
2b7403035459c7 Souptick Joarder 2018-08-23 4143 vm_fault_t do_swap_page(struct vm_fault *vmf)
^1da177e4c3f41 Linus Torvalds 2005-04-16 4144 {
82b0f8c39a3869 Jan Kara 2016-12-14 4145 struct vm_area_struct *vma = vmf->vma;
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4146) struct folio *swapcache, *folio = NULL;
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4147) struct page *page;
2799e77529c2a2 Miaohe Lin 2021-06-28 4148 struct swap_info_struct *si = NULL;
14f9135d547060 David Hildenbrand 2022-05-09 4149 rmap_t rmap_flags = RMAP_NONE;
13ddaf26be324a Kairui Song 2024-02-07 4150 bool need_clear_cache = false;
1493a1913e34b0 David Hildenbrand 2022-05-09 4151 bool exclusive = false;
65500d234e74fc Hugh Dickins 2005-10-29 4152 swp_entry_t entry;
^1da177e4c3f41 Linus Torvalds 2005-04-16 4153 pte_t pte;
2b7403035459c7 Souptick Joarder 2018-08-23 4154 vm_fault_t ret = 0;
aae466b0052e18 Joonsoo Kim 2020-08-11 4155 void *shadow = NULL;
508758960b8d89 Chuanhua Han 2024-05-29 4156 int nr_pages;
508758960b8d89 Chuanhua Han 2024-05-29 4157 unsigned long page_idx;
508758960b8d89 Chuanhua Han 2024-05-29 4158 unsigned long address;
508758960b8d89 Chuanhua Han 2024-05-29 4159 pte_t *ptep;
^1da177e4c3f41 Linus Torvalds 2005-04-16 4160
2ca99358671ad3 Peter Xu 2021-11-05 4161 if (!pte_unmap_same(vmf))
8f4e2101fd7df9 Hugh Dickins 2005-10-29 4162 goto out;
65500d234e74fc Hugh Dickins 2005-10-29 4163
2994302bc8a171 Jan Kara 2016-12-14 4164 entry = pte_to_swp_entry(vmf->orig_pte);
d1737fdbec7f90 Andi Kleen 2009-09-16 4165 if (unlikely(non_swap_entry(entry))) {
0697212a411c1d Christoph Lameter 2006-06-23 4166 if (is_migration_entry(entry)) {
82b0f8c39a3869 Jan Kara 2016-12-14 4167 migration_entry_wait(vma->vm_mm, vmf->pmd,
82b0f8c39a3869 Jan Kara 2016-12-14 4168 vmf->address);
b756a3b5e7ead8 Alistair Popple 2021-06-30 4169 } else if (is_device_exclusive_entry(entry)) {
b756a3b5e7ead8 Alistair Popple 2021-06-30 4170 vmf->page = pfn_swap_entry_to_page(entry);
b756a3b5e7ead8 Alistair Popple 2021-06-30 4171 ret = remove_device_exclusive_entry(vmf);
5042db43cc26f5 Jérôme Glisse 2017-09-08 4172 } else if (is_device_private_entry(entry)) {
1235ccd05b6dd6 Suren Baghdasaryan 2023-06-30 4173 if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
1235ccd05b6dd6 Suren Baghdasaryan 2023-06-30 4174 /*
1235ccd05b6dd6 Suren Baghdasaryan 2023-06-30 4175 * migrate_to_ram is not yet ready to operate
1235ccd05b6dd6 Suren Baghdasaryan 2023-06-30 4176 * under VMA lock.
1235ccd05b6dd6 Suren Baghdasaryan 2023-06-30 4177 */
1235ccd05b6dd6 Suren Baghdasaryan 2023-06-30 4178 vma_end_read(vma);
1235ccd05b6dd6 Suren Baghdasaryan 2023-06-30 4179 ret = VM_FAULT_RETRY;
1235ccd05b6dd6 Suren Baghdasaryan 2023-06-30 4180 goto out;
1235ccd05b6dd6 Suren Baghdasaryan 2023-06-30 4181 }
1235ccd05b6dd6 Suren Baghdasaryan 2023-06-30 4182
af5cdaf82238fb Alistair Popple 2021-06-30 4183 vmf->page = pfn_swap_entry_to_page(entry);
16ce101db85db6 Alistair Popple 2022-09-28 4184 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
16ce101db85db6 Alistair Popple 2022-09-28 4185 vmf->address, &vmf->ptl);
3db82b9374ca92 Hugh Dickins 2023-06-08 4186 if (unlikely(!vmf->pte ||
c33c794828f212 Ryan Roberts 2023-06-12 4187 !pte_same(ptep_get(vmf->pte),
c33c794828f212 Ryan Roberts 2023-06-12 4188 vmf->orig_pte)))
3b65f437d9e8dd Ryan Roberts 2023-06-02 4189 goto unlock;
16ce101db85db6 Alistair Popple 2022-09-28 4190
16ce101db85db6 Alistair Popple 2022-09-28 4191 /*
16ce101db85db6 Alistair Popple 2022-09-28 4192 * Get a page reference while we know the page can't be
16ce101db85db6 Alistair Popple 2022-09-28 4193 * freed.
16ce101db85db6 Alistair Popple 2022-09-28 4194 */
16ce101db85db6 Alistair Popple 2022-09-28 4195 get_page(vmf->page);
16ce101db85db6 Alistair Popple 2022-09-28 4196 pte_unmap_unlock(vmf->pte, vmf->ptl);
4a955bed882e73 Alistair Popple 2022-11-14 4197 ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
16ce101db85db6 Alistair Popple 2022-09-28 4198 put_page(vmf->page);
d1737fdbec7f90 Andi Kleen 2009-09-16 4199 } else if (is_hwpoison_entry(entry)) {
d1737fdbec7f90 Andi Kleen 2009-09-16 4200 ret = VM_FAULT_HWPOISON;
5c041f5d1f23d3 Peter Xu 2022-05-12 4201 } else if (is_pte_marker_entry(entry)) {
5c041f5d1f23d3 Peter Xu 2022-05-12 4202 ret = handle_pte_marker(vmf);
d1737fdbec7f90 Andi Kleen 2009-09-16 4203 } else {
2994302bc8a171 Jan Kara 2016-12-14 4204 print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
d99be1a8ecf377 Hugh Dickins 2009-12-14 4205 ret = VM_FAULT_SIGBUS;
d1737fdbec7f90 Andi Kleen 2009-09-16 4206 }
0697212a411c1d Christoph Lameter 2006-06-23 4207 goto out;
0697212a411c1d Christoph Lameter 2006-06-23 4208 }
0bcac06f27d752 Minchan Kim 2017-11-15 4209
2799e77529c2a2 Miaohe Lin 2021-06-28 4210 /* Prevent swapoff from happening to us. */
2799e77529c2a2 Miaohe Lin 2021-06-28 4211 si = get_swap_device(entry);
2799e77529c2a2 Miaohe Lin 2021-06-28 4212 if (unlikely(!si))
2799e77529c2a2 Miaohe Lin 2021-06-28 4213 goto out;
0bcac06f27d752 Minchan Kim 2017-11-15 4214
5a423081b2465d Matthew Wilcox (Oracle 2022-09-02 4215) folio = swap_cache_get_folio(entry, vma, vmf->address);
5a423081b2465d Matthew Wilcox (Oracle 2022-09-02 4216) if (folio)
5a423081b2465d Matthew Wilcox (Oracle 2022-09-02 4217) page = folio_file_page(folio, swp_offset(entry));
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4218) swapcache = folio;
f80207727aaca3 Minchan Kim 2018-01-18 4219
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4220) if (!folio) {
a449bf58e45abf Qian Cai 2020-08-14 4221 if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
eb085574a7526c Huang Ying 2019-07-11 4222 __swap_count(entry) == 1) {
684d098daf0b3a Chuanhua Han 2024-07-26 4223 /* skip swapcache */
684d098daf0b3a Chuanhua Han 2024-07-26 4224 folio = alloc_swap_folio(vmf);
684d098daf0b3a Chuanhua Han 2024-07-26 4225 page = &folio->page;
684d098daf0b3a Chuanhua Han 2024-07-26 4226 if (folio) {
684d098daf0b3a Chuanhua Han 2024-07-26 4227 __folio_set_locked(folio);
684d098daf0b3a Chuanhua Han 2024-07-26 4228 __folio_set_swapbacked(folio);
684d098daf0b3a Chuanhua Han 2024-07-26 4229
684d098daf0b3a Chuanhua Han 2024-07-26 4230 nr_pages = folio_nr_pages(folio);
nr_pages is initialized here
684d098daf0b3a Chuanhua Han 2024-07-26 4231 if (folio_test_large(folio))
684d098daf0b3a Chuanhua Han 2024-07-26 4232 entry.val = ALIGN_DOWN(entry.val, nr_pages);
13ddaf26be324a Kairui Song 2024-02-07 4233 /*
13ddaf26be324a Kairui Song 2024-02-07 4234 * Prevent parallel swapin from proceeding with
13ddaf26be324a Kairui Song 2024-02-07 4235 * the cache flag. Otherwise, another thread may
13ddaf26be324a Kairui Song 2024-02-07 4236 * finish swapin first, free the entry, and swapout
13ddaf26be324a Kairui Song 2024-02-07 4237 * reusing the same entry. It's undetectable as
13ddaf26be324a Kairui Song 2024-02-07 4238 * pte_same() returns true due to entry reuse.
13ddaf26be324a Kairui Song 2024-02-07 4239 */
684d098daf0b3a Chuanhua Han 2024-07-26 4240 if (swapcache_prepare_nr(entry, nr_pages)) {
13ddaf26be324a Kairui Song 2024-02-07 4241 /* Relax a bit to prevent rapid repeated page faults */
13ddaf26be324a Kairui Song 2024-02-07 4242 schedule_timeout_uninterruptible(1);
684d098daf0b3a Chuanhua Han 2024-07-26 4243 goto out_page;
13ddaf26be324a Kairui Song 2024-02-07 4244 }
13ddaf26be324a Kairui Song 2024-02-07 4245 need_clear_cache = true;
13ddaf26be324a Kairui Song 2024-02-07 4246
6599591816f522 Matthew Wilcox (Oracle 2022-09-02 4247) if (mem_cgroup_swapin_charge_folio(folio,
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4248) vma->vm_mm, GFP_KERNEL,
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4249) entry)) {
545b1b077ca6b3 Michal Hocko 2020-06-25 4250 ret = VM_FAULT_OOM;
4c6355b25e8bb8 Johannes Weiner 2020-06-03 4251 goto out_page;
545b1b077ca6b3 Michal Hocko 2020-06-25 4252 }
684d098daf0b3a Chuanhua Han 2024-07-26 4253 mem_cgroup_swapin_uncharge_swap_nr(entry, nr_pages);
4c6355b25e8bb8 Johannes Weiner 2020-06-03 4254
aae466b0052e18 Joonsoo Kim 2020-08-11 4255 shadow = get_shadow_from_swap_cache(entry);
aae466b0052e18 Joonsoo Kim 2020-08-11 4256 if (shadow)
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4257) workingset_refault(folio, shadow);
0076f029cb2906 Joonsoo Kim 2020-06-25 4258
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4259) folio_add_lru(folio);
0add0c77a9bd0c Shakeel Butt 2021-04-29 4260
c9bdf768dd9319 Matthew Wilcox (Oracle 2023-12-13 4261) /* To provide entry to swap_read_folio() */
3d2c9087688777 David Hildenbrand 2023-08-21 4262 folio->swap = entry;
b2d1f38b524121 Yosry Ahmed 2024-06-07 4263 swap_read_folio(folio, NULL);
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4264) folio->private = NULL;
0bcac06f27d752 Minchan Kim 2017-11-15 4265 }
aa8d22a11da933 Minchan Kim 2017-11-15 4266 } else {
e9e9b7ecee4a13 Minchan Kim 2018-04-05 4267 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
e9e9b7ecee4a13 Minchan Kim 2018-04-05 4268 vmf);
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4269) if (page)
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4270) folio = page_folio(page);
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4271) swapcache = folio;
0bcac06f27d752 Minchan Kim 2017-11-15 4272 }
0bcac06f27d752 Minchan Kim 2017-11-15 4273
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4274) if (!folio) {
^1da177e4c3f41 Linus Torvalds 2005-04-16 4275 /*
8f4e2101fd7df9 Hugh Dickins 2005-10-29 4276 * Back out if somebody else faulted in this pte
8f4e2101fd7df9 Hugh Dickins 2005-10-29 4277 * while we released the pte lock.
^1da177e4c3f41 Linus Torvalds 2005-04-16 4278 */
82b0f8c39a3869 Jan Kara 2016-12-14 4279 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
82b0f8c39a3869 Jan Kara 2016-12-14 4280 vmf->address, &vmf->ptl);
c33c794828f212 Ryan Roberts 2023-06-12 4281 if (likely(vmf->pte &&
c33c794828f212 Ryan Roberts 2023-06-12 4282 pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
^1da177e4c3f41 Linus Torvalds 2005-04-16 4283 ret = VM_FAULT_OOM;
65500d234e74fc Hugh Dickins 2005-10-29 4284 goto unlock;
^1da177e4c3f41 Linus Torvalds 2005-04-16 4285 }
^1da177e4c3f41 Linus Torvalds 2005-04-16 4286
^1da177e4c3f41 Linus Torvalds 2005-04-16 4287 /* Had to read the page from swap area: Major fault */
^1da177e4c3f41 Linus Torvalds 2005-04-16 4288 ret = VM_FAULT_MAJOR;
f8891e5e1f93a1 Christoph Lameter 2006-06-30 4289 count_vm_event(PGMAJFAULT);
2262185c5b287f Roman Gushchin 2017-07-06 4290 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
d1737fdbec7f90 Andi Kleen 2009-09-16 4291 } else if (PageHWPoison(page)) {
71f72525dfaaec Wu Fengguang 2009-12-16 4292 /*
71f72525dfaaec Wu Fengguang 2009-12-16 4293 * hwpoisoned dirty swapcache pages are kept for killing
71f72525dfaaec Wu Fengguang 2009-12-16 4294 * owner processes (which may be unknown at hwpoison time)
71f72525dfaaec Wu Fengguang 2009-12-16 4295 */
d1737fdbec7f90 Andi Kleen 2009-09-16 4296 ret = VM_FAULT_HWPOISON;
4779cb31c0ee3b Andi Kleen 2009-10-14 4297 goto out_release;
^1da177e4c3f41 Linus Torvalds 2005-04-16 4298 }
^1da177e4c3f41 Linus Torvalds 2005-04-16 4299
fdc724d6aa44ef Suren Baghdasaryan 2023-06-30 4300 ret |= folio_lock_or_retry(folio, vmf);
fdc724d6aa44ef Suren Baghdasaryan 2023-06-30 4301 if (ret & VM_FAULT_RETRY)
d065bd810b6deb Michel Lespinasse 2010-10-26 4302 goto out_release;
073e587ec2cc37 KAMEZAWA Hiroyuki 2008-10-18 4303
84d60fdd3733fb David Hildenbrand 2022-03-24 4304 if (swapcache) {
4969c1192d15af Andrea Arcangeli 2010-09-09 4305 /*
3b344157c0c15b Matthew Wilcox (Oracle 2022-09-02 4306) * Make sure folio_free_swap() or swapoff did not release the
84d60fdd3733fb David Hildenbrand 2022-03-24 4307 * swapcache from under us. The page pin, and pte_same test
84d60fdd3733fb David Hildenbrand 2022-03-24 4308 * below, are not enough to exclude that. Even if it is still
84d60fdd3733fb David Hildenbrand 2022-03-24 4309 * swapcache, we need to check that the page's swap has not
84d60fdd3733fb David Hildenbrand 2022-03-24 4310 * changed.
4969c1192d15af Andrea Arcangeli 2010-09-09 4311 */
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4312) if (unlikely(!folio_test_swapcache(folio) ||
cfeed8ffe55b37 David Hildenbrand 2023-08-21 4313 page_swap_entry(page).val != entry.val))
4969c1192d15af Andrea Arcangeli 2010-09-09 4314 goto out_page;
4969c1192d15af Andrea Arcangeli 2010-09-09 4315
84d60fdd3733fb David Hildenbrand 2022-03-24 4316 /*
84d60fdd3733fb David Hildenbrand 2022-03-24 4317 * KSM sometimes has to copy on read faults, for example, if
84d60fdd3733fb David Hildenbrand 2022-03-24 4318 * page->index of !PageKSM() pages would be nonlinear inside the
84d60fdd3733fb David Hildenbrand 2022-03-24 4319 * anon VMA -- PageKSM() is lost on actual swapout.
84d60fdd3733fb David Hildenbrand 2022-03-24 4320 */
96db66d9c8f3c1 Matthew Wilcox (Oracle 2023-12-11 4321) folio = ksm_might_need_to_copy(folio, vma, vmf->address);
96db66d9c8f3c1 Matthew Wilcox (Oracle 2023-12-11 4322) if (unlikely(!folio)) {
5ad6468801d28c Hugh Dickins 2009-12-14 4323 ret = VM_FAULT_OOM;
96db66d9c8f3c1 Matthew Wilcox (Oracle 2023-12-11 4324) folio = swapcache;
4969c1192d15af Andrea Arcangeli 2010-09-09 4325 goto out_page;
96db66d9c8f3c1 Matthew Wilcox (Oracle 2023-12-11 4326) } else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
6b970599e807ea Kefeng Wang 2022-12-09 4327 ret = VM_FAULT_HWPOISON;
96db66d9c8f3c1 Matthew Wilcox (Oracle 2023-12-11 4328) folio = swapcache;
6b970599e807ea Kefeng Wang 2022-12-09 4329 goto out_page;
4969c1192d15af Andrea Arcangeli 2010-09-09 4330 }
96db66d9c8f3c1 Matthew Wilcox (Oracle 2023-12-11 4331) if (folio != swapcache)
96db66d9c8f3c1 Matthew Wilcox (Oracle 2023-12-11 4332) page = folio_page(folio, 0);
c145e0b47c77eb David Hildenbrand 2022-03-24 4333
c145e0b47c77eb David Hildenbrand 2022-03-24 4334 /*
c145e0b47c77eb David Hildenbrand 2022-03-24 4335 * If we want to map a page that's in the swapcache writable, we
c145e0b47c77eb David Hildenbrand 2022-03-24 4336 * have to detect via the refcount if we're really the exclusive
c145e0b47c77eb David Hildenbrand 2022-03-24 4337 * owner. Try removing the extra reference from the local LRU
1fec6890bf2247 Matthew Wilcox (Oracle 2023-06-21 4338) * caches if required.
c145e0b47c77eb David Hildenbrand 2022-03-24 4339 */
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4340) if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4341) !folio_test_ksm(folio) && !folio_test_lru(folio))
c145e0b47c77eb David Hildenbrand 2022-03-24 4342 lru_add_drain();
84d60fdd3733fb David Hildenbrand 2022-03-24 4343 }
5ad6468801d28c Hugh Dickins 2009-12-14 4344
4231f8425833b1 Kefeng Wang 2023-03-02 4345 folio_throttle_swaprate(folio, GFP_KERNEL);
8a9f3ccd24741b Balbir Singh 2008-02-07 4346
^1da177e4c3f41 Linus Torvalds 2005-04-16 4347 /*
8f4e2101fd7df9 Hugh Dickins 2005-10-29 4348 * Back out if somebody else already faulted in this pte.
^1da177e4c3f41 Linus Torvalds 2005-04-16 4349 */
82b0f8c39a3869 Jan Kara 2016-12-14 4350 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
82b0f8c39a3869 Jan Kara 2016-12-14 4351 &vmf->ptl);
c33c794828f212 Ryan Roberts 2023-06-12 4352 if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
b81074800b98ac Kirill Korotaev 2005-05-16 4353 goto out_nomap;
b81074800b98ac Kirill Korotaev 2005-05-16 4354
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4355) if (unlikely(!folio_test_uptodate(folio))) {
b81074800b98ac Kirill Korotaev 2005-05-16 4356 ret = VM_FAULT_SIGBUS;
b81074800b98ac Kirill Korotaev 2005-05-16 4357 goto out_nomap;
^1da177e4c3f41 Linus Torvalds 2005-04-16 4358 }
^1da177e4c3f41 Linus Torvalds 2005-04-16 4359
684d098daf0b3a Chuanhua Han 2024-07-26 4360 /* allocated large folios for SWP_SYNCHRONOUS_IO */
684d098daf0b3a Chuanhua Han 2024-07-26 4361 if (folio_test_large(folio) && !folio_test_swapcache(folio)) {
684d098daf0b3a Chuanhua Han 2024-07-26 4362 unsigned long nr = folio_nr_pages(folio);
684d098daf0b3a Chuanhua Han 2024-07-26 4363 unsigned long folio_start = ALIGN_DOWN(vmf->address, nr * PAGE_SIZE);
684d098daf0b3a Chuanhua Han 2024-07-26 4364 unsigned long idx = (vmf->address - folio_start) / PAGE_SIZE;
684d098daf0b3a Chuanhua Han 2024-07-26 4365 pte_t *folio_ptep = vmf->pte - idx;
684d098daf0b3a Chuanhua Han 2024-07-26 4366
684d098daf0b3a Chuanhua Han 2024-07-26 4367 if (!can_swapin_thp(vmf, folio_ptep, nr))
684d098daf0b3a Chuanhua Han 2024-07-26 4368 goto out_nomap;
684d098daf0b3a Chuanhua Han 2024-07-26 4369
684d098daf0b3a Chuanhua Han 2024-07-26 4370 page_idx = idx;
684d098daf0b3a Chuanhua Han 2024-07-26 4371 address = folio_start;
684d098daf0b3a Chuanhua Han 2024-07-26 4372 ptep = folio_ptep;
684d098daf0b3a Chuanhua Han 2024-07-26 4373 goto check_folio;
Let's say we hit this goto
684d098daf0b3a Chuanhua Han 2024-07-26 4374 }
684d098daf0b3a Chuanhua Han 2024-07-26 4375
508758960b8d89 Chuanhua Han 2024-05-29 4376 nr_pages = 1;
508758960b8d89 Chuanhua Han 2024-05-29 4377 page_idx = 0;
508758960b8d89 Chuanhua Han 2024-05-29 4378 address = vmf->address;
508758960b8d89 Chuanhua Han 2024-05-29 4379 ptep = vmf->pte;
508758960b8d89 Chuanhua Han 2024-05-29 4380 if (folio_test_large(folio) && folio_test_swapcache(folio)) {
508758960b8d89 Chuanhua Han 2024-05-29 4381 int nr = folio_nr_pages(folio);
508758960b8d89 Chuanhua Han 2024-05-29 4382 unsigned long idx = folio_page_idx(folio, page);
508758960b8d89 Chuanhua Han 2024-05-29 4383 unsigned long folio_start = address - idx * PAGE_SIZE;
508758960b8d89 Chuanhua Han 2024-05-29 4384 unsigned long folio_end = folio_start + nr * PAGE_SIZE;
508758960b8d89 Chuanhua Han 2024-05-29 4385 pte_t *folio_ptep;
508758960b8d89 Chuanhua Han 2024-05-29 4386 pte_t folio_pte;
508758960b8d89 Chuanhua Han 2024-05-29 4387
508758960b8d89 Chuanhua Han 2024-05-29 4388 if (unlikely(folio_start < max(address & PMD_MASK, vma->vm_start)))
508758960b8d89 Chuanhua Han 2024-05-29 4389 goto check_folio;
508758960b8d89 Chuanhua Han 2024-05-29 4390 if (unlikely(folio_end > pmd_addr_end(address, vma->vm_end)))
508758960b8d89 Chuanhua Han 2024-05-29 4391 goto check_folio;
508758960b8d89 Chuanhua Han 2024-05-29 4392
508758960b8d89 Chuanhua Han 2024-05-29 4393 folio_ptep = vmf->pte - idx;
508758960b8d89 Chuanhua Han 2024-05-29 4394 folio_pte = ptep_get(folio_ptep);
508758960b8d89 Chuanhua Han 2024-05-29 4395 if (!pte_same(folio_pte, pte_move_swp_offset(vmf->orig_pte, -idx)) ||
508758960b8d89 Chuanhua Han 2024-05-29 4396 swap_pte_batch(folio_ptep, nr, folio_pte) != nr)
508758960b8d89 Chuanhua Han 2024-05-29 4397 goto check_folio;
508758960b8d89 Chuanhua Han 2024-05-29 4398
508758960b8d89 Chuanhua Han 2024-05-29 4399 page_idx = idx;
508758960b8d89 Chuanhua Han 2024-05-29 4400 address = folio_start;
508758960b8d89 Chuanhua Han 2024-05-29 4401 ptep = folio_ptep;
508758960b8d89 Chuanhua Han 2024-05-29 4402 nr_pages = nr;
508758960b8d89 Chuanhua Han 2024-05-29 4403 entry = folio->swap;
508758960b8d89 Chuanhua Han 2024-05-29 4404 page = &folio->page;
508758960b8d89 Chuanhua Han 2024-05-29 4405 }
508758960b8d89 Chuanhua Han 2024-05-29 4406
508758960b8d89 Chuanhua Han 2024-05-29 4407 check_folio:
78fbe906cc900b David Hildenbrand 2022-05-09 4408 /*
78fbe906cc900b David Hildenbrand 2022-05-09 4409 * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte
78fbe906cc900b David Hildenbrand 2022-05-09 4410 * must never point at an anonymous page in the swapcache that is
78fbe906cc900b David Hildenbrand 2022-05-09 4411 * PG_anon_exclusive. Sanity check that this holds and especially, that
78fbe906cc900b David Hildenbrand 2022-05-09 4412 * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity
78fbe906cc900b David Hildenbrand 2022-05-09 4413 * check after taking the PT lock and making sure that nobody
78fbe906cc900b David Hildenbrand 2022-05-09 4414 * concurrently faulted in this page and set PG_anon_exclusive.
78fbe906cc900b David Hildenbrand 2022-05-09 4415 */
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4416) BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio));
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4417) BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));
78fbe906cc900b David Hildenbrand 2022-05-09 4418
1493a1913e34b0 David Hildenbrand 2022-05-09 4419 /*
1493a1913e34b0 David Hildenbrand 2022-05-09 4420 * Check under PT lock (to protect against concurrent fork() sharing
1493a1913e34b0 David Hildenbrand 2022-05-09 4421 * the swap entry concurrently) for certainly exclusive pages.
1493a1913e34b0 David Hildenbrand 2022-05-09 4422 */
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4423) if (!folio_test_ksm(folio)) {
1493a1913e34b0 David Hildenbrand 2022-05-09 4424 exclusive = pte_swp_exclusive(vmf->orig_pte);
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4425) if (folio != swapcache) {
1493a1913e34b0 David Hildenbrand 2022-05-09 4426 /*
1493a1913e34b0 David Hildenbrand 2022-05-09 4427 * We have a fresh page that is not exposed to the
1493a1913e34b0 David Hildenbrand 2022-05-09 4428 * swapcache -> certainly exclusive.
1493a1913e34b0 David Hildenbrand 2022-05-09 4429 */
1493a1913e34b0 David Hildenbrand 2022-05-09 4430 exclusive = true;
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4431) } else if (exclusive && folio_test_writeback(folio) &&
eacde32757c756 Miaohe Lin 2022-05-19 4432 data_race(si->flags & SWP_STABLE_WRITES)) {
1493a1913e34b0 David Hildenbrand 2022-05-09 4433 /*
1493a1913e34b0 David Hildenbrand 2022-05-09 4434 * This is tricky: not all swap backends support
1493a1913e34b0 David Hildenbrand 2022-05-09 4435 * concurrent page modifications while under writeback.
1493a1913e34b0 David Hildenbrand 2022-05-09 4436 *
1493a1913e34b0 David Hildenbrand 2022-05-09 4437 * So if we stumble over such a page in the swapcache
1493a1913e34b0 David Hildenbrand 2022-05-09 4438 * we must not set the page exclusive, otherwise we can
1493a1913e34b0 David Hildenbrand 2022-05-09 4439 * map it writable without further checks and modify it
1493a1913e34b0 David Hildenbrand 2022-05-09 4440 * while still under writeback.
1493a1913e34b0 David Hildenbrand 2022-05-09 4441 *
1493a1913e34b0 David Hildenbrand 2022-05-09 4442 * For these problematic swap backends, simply drop the
1493a1913e34b0 David Hildenbrand 2022-05-09 4443 * exclusive marker: this is perfectly fine as we start
1493a1913e34b0 David Hildenbrand 2022-05-09 4444 * writeback only if we fully unmapped the page and
1493a1913e34b0 David Hildenbrand 2022-05-09 4445 * there are no unexpected references on the page after
1493a1913e34b0 David Hildenbrand 2022-05-09 4446 * unmapping succeeded. After fully unmapped, no
1493a1913e34b0 David Hildenbrand 2022-05-09 4447 * further GUP references (FOLL_GET and FOLL_PIN) can
1493a1913e34b0 David Hildenbrand 2022-05-09 4448 * appear, so dropping the exclusive marker and mapping
1493a1913e34b0 David Hildenbrand 2022-05-09 4449 * it only R/O is fine.
1493a1913e34b0 David Hildenbrand 2022-05-09 4450 */
1493a1913e34b0 David Hildenbrand 2022-05-09 4451 exclusive = false;
1493a1913e34b0 David Hildenbrand 2022-05-09 4452 }
1493a1913e34b0 David Hildenbrand 2022-05-09 4453 }
1493a1913e34b0 David Hildenbrand 2022-05-09 4454
6dca4ac6fc91fd Peter Collingbourne 2023-05-22 4455 /*
6dca4ac6fc91fd Peter Collingbourne 2023-05-22 4456 * Some architectures may have to restore extra metadata to the page
6dca4ac6fc91fd Peter Collingbourne 2023-05-22 4457 * when reading from swap. This metadata may be indexed by swap entry
6dca4ac6fc91fd Peter Collingbourne 2023-05-22 4458 * so this must be called before swap_free().
6dca4ac6fc91fd Peter Collingbourne 2023-05-22 4459 */
f238b8c33c6738 Barry Song 2024-03-23 4460 arch_swap_restore(folio_swap(entry, folio), folio);
6dca4ac6fc91fd Peter Collingbourne 2023-05-22 4461
8c7c6e34a1256a KAMEZAWA Hiroyuki 2009-01-07 4462 /*
c145e0b47c77eb David Hildenbrand 2022-03-24 4463 * Remove the swap entry and conditionally try to free up the swapcache.
c145e0b47c77eb David Hildenbrand 2022-03-24 4464 * We're already holding a reference on the page but haven't mapped it
c145e0b47c77eb David Hildenbrand 2022-03-24 4465 * yet.
8c7c6e34a1256a KAMEZAWA Hiroyuki 2009-01-07 4466 */
508758960b8d89 Chuanhua Han 2024-05-29 @4467 swap_free_nr(entry, nr_pages);
^^^^^^^^
Smatch warning. The code is a bit complicated so it could be a false
positive.
a160e5377b55bc Matthew Wilcox (Oracle 2022-09-02 4468) if (should_try_to_free_swap(folio, vma, vmf->flags))
a160e5377b55bc Matthew Wilcox (Oracle 2022-09-02 4469) folio_free_swap(folio);
^1da177e4c3f41 Linus Torvalds 2005-04-16 4470
508758960b8d89 Chuanhua Han 2024-05-29 4471 add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages);
508758960b8d89 Chuanhua Han 2024-05-29 4472 add_mm_counter(vma->vm_mm, MM_SWAPENTS, -nr_pages);
^1da177e4c3f41 Linus Torvalds 2005-04-16 4473 pte = mk_pte(page, vma->vm_page_prot);
c18160dba5ff63 Barry Song 2024-06-02 4474 if (pte_swp_soft_dirty(vmf->orig_pte))
c18160dba5ff63 Barry Song 2024-06-02 4475 pte = pte_mksoft_dirty(pte);
c18160dba5ff63 Barry Song 2024-06-02 4476 if (pte_swp_uffd_wp(vmf->orig_pte))
c18160dba5ff63 Barry Song 2024-06-02 4477 pte = pte_mkuffd_wp(pte);
c145e0b47c77eb David Hildenbrand 2022-03-24 4478
c145e0b47c77eb David Hildenbrand 2022-03-24 4479 /*
1493a1913e34b0 David Hildenbrand 2022-05-09 4480 * Same logic as in do_wp_page(); however, optimize for pages that are
1493a1913e34b0 David Hildenbrand 2022-05-09 4481 * certainly not shared either because we just allocated them without
1493a1913e34b0 David Hildenbrand 2022-05-09 4482 * exposing them to the swapcache or because the swap entry indicates
1493a1913e34b0 David Hildenbrand 2022-05-09 4483 * exclusivity.
c145e0b47c77eb David Hildenbrand 2022-03-24 4484 */
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4485) if (!folio_test_ksm(folio) &&
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4486) (exclusive || folio_ref_count(folio) == 1)) {
c18160dba5ff63 Barry Song 2024-06-02 4487 if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) &&
20dfa5b7adc5a1 Barry Song 2024-06-08 4488 !pte_needs_soft_dirty_wp(vma, pte)) {
c18160dba5ff63 Barry Song 2024-06-02 4489 pte = pte_mkwrite(pte, vma);
6c287605fd5646 David Hildenbrand 2022-05-09 4490 if (vmf->flags & FAULT_FLAG_WRITE) {
c18160dba5ff63 Barry Song 2024-06-02 4491 pte = pte_mkdirty(pte);
82b0f8c39a3869 Jan Kara 2016-12-14 4492 vmf->flags &= ~FAULT_FLAG_WRITE;
6c287605fd5646 David Hildenbrand 2022-05-09 4493 }
c18160dba5ff63 Barry Song 2024-06-02 4494 }
14f9135d547060 David Hildenbrand 2022-05-09 4495 rmap_flags |= RMAP_EXCLUSIVE;
^1da177e4c3f41 Linus Torvalds 2005-04-16 4496 }
508758960b8d89 Chuanhua Han 2024-05-29 4497 folio_ref_add(folio, nr_pages - 1);
508758960b8d89 Chuanhua Han 2024-05-29 4498 flush_icache_pages(vma, page, nr_pages);
508758960b8d89 Chuanhua Han 2024-05-29 4499 vmf->orig_pte = pte_advance_pfn(pte, page_idx);
0bcac06f27d752 Minchan Kim 2017-11-15 4500
0bcac06f27d752 Minchan Kim 2017-11-15 4501 /* ksm created a completely new copy */
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4502) if (unlikely(folio != swapcache && swapcache)) {
15bde4abab734c Barry Song 2024-06-18 4503 folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE);
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4504) folio_add_lru_vma(folio, vma);
9ae2feacedde16 Barry Song 2024-06-18 4505 } else if (!folio_test_anon(folio)) {
9ae2feacedde16 Barry Song 2024-06-18 4506 /*
684d098daf0b3a Chuanhua Han 2024-07-26 4507 * We currently only expect small !anon folios which are either
684d098daf0b3a Chuanhua Han 2024-07-26 4508 * fully exclusive or fully shared, or new allocated large folios
684d098daf0b3a Chuanhua Han 2024-07-26 4509 * which are fully exclusive. If we ever get large folios within
684d098daf0b3a Chuanhua Han 2024-07-26 4510 * swapcache here, we have to be careful.
9ae2feacedde16 Barry Song 2024-06-18 4511 */
684d098daf0b3a Chuanhua Han 2024-07-26 4512 VM_WARN_ON_ONCE(folio_test_large(folio) && folio_test_swapcache(folio));
9ae2feacedde16 Barry Song 2024-06-18 4513 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
9ae2feacedde16 Barry Song 2024-06-18 4514 folio_add_new_anon_rmap(folio, vma, address, rmap_flags);
0bcac06f27d752 Minchan Kim 2017-11-15 4515 } else {
508758960b8d89 Chuanhua Han 2024-05-29 4516 folio_add_anon_rmap_ptes(folio, page, nr_pages, vma, address,
b832a354d787bf David Hildenbrand 2023-12-20 4517 rmap_flags);
00501b531c4723 Johannes Weiner 2014-08-08 4518 }
^1da177e4c3f41 Linus Torvalds 2005-04-16 4519
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4520) VM_BUG_ON(!folio_test_anon(folio) ||
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4521) (pte_write(pte) && !PageAnonExclusive(page)));
508758960b8d89 Chuanhua Han 2024-05-29 4522 set_ptes(vma->vm_mm, address, ptep, pte, nr_pages);
508758960b8d89 Chuanhua Han 2024-05-29 4523 arch_do_swap_page_nr(vma->vm_mm, vma, address,
508758960b8d89 Chuanhua Han 2024-05-29 4524 pte, pte, nr_pages);
1eba86c096e35e Pasha Tatashin 2022-01-14 4525
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4526) folio_unlock(folio);
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4527) if (folio != swapcache && swapcache) {
4969c1192d15af Andrea Arcangeli 2010-09-09 4528 /*
4969c1192d15af Andrea Arcangeli 2010-09-09 4529 * Hold the lock to avoid the swap entry to be reused
4969c1192d15af Andrea Arcangeli 2010-09-09 4530 * until we take the PT lock for the pte_same() check
4969c1192d15af Andrea Arcangeli 2010-09-09 4531 * (to avoid false positives from pte_same). For
4969c1192d15af Andrea Arcangeli 2010-09-09 4532 * further safety release the lock after the swap_free
4969c1192d15af Andrea Arcangeli 2010-09-09 4533 * so that the swap count won't change under a
4969c1192d15af Andrea Arcangeli 2010-09-09 4534 * parallel locked swapcache.
4969c1192d15af Andrea Arcangeli 2010-09-09 4535 */
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4536) folio_unlock(swapcache);
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4537) folio_put(swapcache);
4969c1192d15af Andrea Arcangeli 2010-09-09 4538 }
c475a8ab625d56 Hugh Dickins 2005-06-21 4539
82b0f8c39a3869 Jan Kara 2016-12-14 4540 if (vmf->flags & FAULT_FLAG_WRITE) {
2994302bc8a171 Jan Kara 2016-12-14 4541 ret |= do_wp_page(vmf);
61469f1d51777f Hugh Dickins 2008-03-04 4542 if (ret & VM_FAULT_ERROR)
61469f1d51777f Hugh Dickins 2008-03-04 4543 ret &= VM_FAULT_ERROR;
^1da177e4c3f41 Linus Torvalds 2005-04-16 4544 goto out;
^1da177e4c3f41 Linus Torvalds 2005-04-16 4545 }
^1da177e4c3f41 Linus Torvalds 2005-04-16 4546
^1da177e4c3f41 Linus Torvalds 2005-04-16 4547 /* No need to invalidate - it was non-present before */
508758960b8d89 Chuanhua Han 2024-05-29 4548 update_mmu_cache_range(vmf, vma, address, ptep, nr_pages);
65500d234e74fc Hugh Dickins 2005-10-29 4549 unlock:
3db82b9374ca92 Hugh Dickins 2023-06-08 4550 if (vmf->pte)
82b0f8c39a3869 Jan Kara 2016-12-14 4551 pte_unmap_unlock(vmf->pte, vmf->ptl);
^1da177e4c3f41 Linus Torvalds 2005-04-16 4552 out:
13ddaf26be324a Kairui Song 2024-02-07 4553 /* Clear the swap cache pin for direct swapin after PTL unlock */
13ddaf26be324a Kairui Song 2024-02-07 4554 if (need_clear_cache)
684d098daf0b3a Chuanhua Han 2024-07-26 4555 swapcache_clear_nr(si, entry, nr_pages);
2799e77529c2a2 Miaohe Lin 2021-06-28 4556 if (si)
2799e77529c2a2 Miaohe Lin 2021-06-28 4557 put_swap_device(si);
^1da177e4c3f41 Linus Torvalds 2005-04-16 4558 return ret;
b81074800b98ac Kirill Korotaev 2005-05-16 4559 out_nomap:
3db82b9374ca92 Hugh Dickins 2023-06-08 4560 if (vmf->pte)
82b0f8c39a3869 Jan Kara 2016-12-14 4561 pte_unmap_unlock(vmf->pte, vmf->ptl);
bc43f75cd98158 Johannes Weiner 2009-04-30 4562 out_page:
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4563) folio_unlock(folio);
4779cb31c0ee3b Andi Kleen 2009-10-14 4564 out_release:
63ad4add382305 Matthew Wilcox (Oracle 2022-09-02 4565) folio_put(folio);
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4566) if (folio != swapcache && swapcache) {
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4567) folio_unlock(swapcache);
d4f9565ae598bd Matthew Wilcox (Oracle 2022-09-02 4568) folio_put(swapcache);
4969c1192d15af Andrea Arcangeli 2010-09-09 4569 }
13ddaf26be324a Kairui Song 2024-02-07 4570 if (need_clear_cache)
684d098daf0b3a Chuanhua Han 2024-07-26 4571 swapcache_clear_nr(si, entry, nr_pages);
2799e77529c2a2 Miaohe Lin 2021-06-28 4572 if (si)
2799e77529c2a2 Miaohe Lin 2021-06-28 4573 put_swap_device(si);
65500d234e74fc Hugh Dickins 2005-10-29 4574 return ret;
^1da177e4c3f41 Linus Torvalds 2005-04-16 4575 }
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki