/* $NetBSD: uvm_pager.c,v 1.131 2024/03/15 07:09:37 andvar Exp $ */ /* * Copyright (c) 1997 Charles D. Cranor and Washington University. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp */ /* * uvm_pager.c: generic functions used to assist the pagers. */ #include __KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.131 2024/03/15 07:09:37 andvar Exp $"); #include "opt_uvmhist.h" #include "opt_readahead.h" #include "opt_pagermap.h" #include #include #include #include #include #include /* * XXX * this is needed until the device strategy interface * is changed to do physically-addressed i/o. */ #ifndef PAGER_MAP_DEFAULT_SIZE #define PAGER_MAP_DEFAULT_SIZE (16 * 1024 * 1024) #endif #ifndef PAGER_MAP_SIZE #define PAGER_MAP_SIZE PAGER_MAP_DEFAULT_SIZE #endif size_t pager_map_size = PAGER_MAP_SIZE; /* * list of uvm pagers in the system */ const struct uvm_pagerops * const uvmpagerops[] = { &aobj_pager, &uvm_deviceops, &uvm_vnodeops, &ubc_pager, }; /* * the pager map: provides KVA for I/O */ struct vm_map *pager_map; /* XXX */ kmutex_t pager_map_wanted_lock __cacheline_aligned; bool pager_map_wanted; /* locked by pager map */ static vaddr_t emergva; static int emerg_ncolors; static bool emerginuse; void uvm_pager_realloc_emerg(void) { vaddr_t new_emergva, old_emergva; int old_emerg_ncolors; if (__predict_true(emergva != 0 && emerg_ncolors >= uvmexp.ncolors)) return; KASSERT(!emerginuse); new_emergva = uvm_km_alloc(kernel_map, round_page(MAXPHYS) + ptoa(uvmexp.ncolors), ptoa(uvmexp.ncolors), UVM_KMF_VAONLY); KASSERT(new_emergva != 0); old_emergva = emergva; old_emerg_ncolors = emerg_ncolors; /* * don't support re-color in late boot anyway. */ if (0) /* XXX */ mutex_enter(&pager_map_wanted_lock); emergva = new_emergva; emerg_ncolors = uvmexp.ncolors; wakeup(&old_emergva); if (0) /* XXX */ mutex_exit(&pager_map_wanted_lock); if (old_emergva) uvm_km_free(kernel_map, old_emergva, round_page(MAXPHYS) + ptoa(old_emerg_ncolors), UVM_KMF_VAONLY); } /* * uvm_pager_init: init pagers (at boot time) */ void uvm_pager_init(void) { u_int lcv; vaddr_t sva, eva; /* * init pager map */ sva = 0; pager_map = uvm_km_suballoc(kernel_map, &sva, &eva, pager_map_size, 0, false, NULL); mutex_init(&pager_map_wanted_lock, MUTEX_DEFAULT, IPL_NONE); pager_map_wanted = false; uvm_pager_realloc_emerg(); /* * call pager init functions */ for (lcv = 0 ; lcv < __arraycount(uvmpagerops); lcv++) { if (uvmpagerops[lcv]->pgo_init) uvmpagerops[lcv]->pgo_init(); } } #ifdef PMAP_DIRECT /* * uvm_pagermapdirect: map a single page via the pmap's direct segment * * this is an abuse of pmap_direct_process(), since the kva is being grabbed * and no processing is taking place, but for now.. */ static int uvm_pagermapdirect(void *kva, size_t sz, void *cookie) { KASSERT(sz == PAGE_SIZE); *(vaddr_t *)cookie = (vaddr_t)kva; return 0; } #endif /* * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings * * we basically just map in a blank map entry to reserve the space in the * map and then use pmap_enter() to put the mappings in by hand. */ vaddr_t uvm_pagermapin(struct vm_page **pps, int npages, int flags) { vsize_t size; vaddr_t kva; vaddr_t cva; struct vm_page *pp; vm_prot_t prot; const bool pdaemon = (curlwp == uvm.pagedaemon_lwp); const u_int first_color = VM_PGCOLOR(*pps); UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(maphist,"(pps=%#jx, npages=%jd, first_color=%ju)", (uintptr_t)pps, npages, first_color, 0); #ifdef PMAP_DIRECT /* * for a single page the direct mapped segment can be used. */ if (npages == 1) { int error __diagused; KASSERT((pps[0]->flags & PG_BUSY) != 0); error = pmap_direct_process(VM_PAGE_TO_PHYS(pps[0]), 0, PAGE_SIZE, uvm_pagermapdirect, &kva); KASSERT(error == 0); UVMHIST_LOG(maphist, "<- done, direct (KVA=%#jx)", kva,0,0,0); return kva; } #endif /* * compute protection. outgoing I/O only needs read * access to the page, whereas incoming needs read/write. */ prot = VM_PROT_READ; if (flags & UVMPAGER_MAPIN_READ) prot |= VM_PROT_WRITE; ReStart: size = ptoa(npages); kva = 0; /* let system choose VA */ if (uvm_map(pager_map, &kva, size, NULL, UVM_UNKNOWN_OFFSET, first_color, UVM_FLAG_COLORMATCH | UVM_FLAG_NOMERGE | (pdaemon ? UVM_FLAG_NOWAIT : 0)) != 0) { if (pdaemon) { mutex_enter(&pager_map_wanted_lock); if (emerginuse) { UVM_UNLOCK_AND_WAIT(&emergva, &pager_map_wanted_lock, false, "emergva", 0); goto ReStart; } emerginuse = true; mutex_exit(&pager_map_wanted_lock); kva = emergva + ptoa(first_color); /* The shift implicitly truncates to PAGE_SIZE */ KASSERT(npages <= (MAXPHYS >> PAGE_SHIFT)); goto enter; } if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) { UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0); return(0); } mutex_enter(&pager_map_wanted_lock); pager_map_wanted = true; UVMHIST_LOG(maphist, " SLEEPING on pager_map",0,0,0,0); UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, false, "pager_map", 0); goto ReStart; } enter: /* got it */ for (cva = kva; npages != 0; npages--, cva += PAGE_SIZE) { pp = *pps++; KASSERT(pp); // KASSERT(!((VM_PAGE_TO_PHYS(pp) ^ cva) & uvmexp.colormask)); KASSERT(pp->flags & PG_BUSY); pmap_kenter_pa(cva, VM_PAGE_TO_PHYS(pp), prot, 0); } pmap_update(vm_map_pmap(pager_map)); UVMHIST_LOG(maphist, "<- done (KVA=%#jx)", kva,0,0,0); return(kva); } /* * uvm_pagermapout: remove pager_map mapping * * we remove our mappings by hand and then remove the mapping (waking * up anyone wanting space). */ void uvm_pagermapout(vaddr_t kva, int npages) { vsize_t size = ptoa(npages); struct vm_map_entry *entries; UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(maphist, " (kva=%#jx, npages=%jd)", kva, npages,0,0); #ifdef PMAP_DIRECT /* * solitary pages are mapped directly. */ if (npages == 1) { UVMHIST_LOG(maphist,"<- done, direct", 0,0,0,0); return; } #endif /* * duplicate uvm_unmap, but add in pager_map_wanted handling. */ pmap_kremove(kva, size); pmap_update(pmap_kernel()); if ((kva & ~ptoa(uvmexp.colormask)) == emergva) { mutex_enter(&pager_map_wanted_lock); KASSERT(emerginuse); emerginuse = false; wakeup(&emergva); mutex_exit(&pager_map_wanted_lock); return; } vm_map_lock(pager_map); uvm_unmap_remove(pager_map, kva, kva + size, &entries, 0); mutex_enter(&pager_map_wanted_lock); if (pager_map_wanted) { pager_map_wanted = false; wakeup(pager_map); } mutex_exit(&pager_map_wanted_lock); vm_map_unlock(pager_map); if (entries) uvm_unmap_detach(entries, 0); UVMHIST_LOG(maphist,"<- done",0,0,0,0); } void uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error) { struct uvm_object *uobj; struct vm_page *pg; krwlock_t *slock; int pageout_done; /* number of PG_PAGEOUT pages processed */ int swslot __unused; /* used for VMSWAP */ int i; bool swap; UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); swslot = 0; pageout_done = 0; slock = NULL; uobj = NULL; pg = pgs[0]; swap = (pg->uanon != NULL && pg->uobject == NULL) || (pg->flags & PG_AOBJ) != 0; if (!swap) { uobj = pg->uobject; slock = uobj->vmobjlock; rw_enter(slock, RW_WRITER); } else { #if defined(VMSWAP) if (error) { if (pg->uobject != NULL) { swslot = uao_find_swslot(pg->uobject, pg->offset >> PAGE_SHIFT); } else { KASSERT(pg->uanon != NULL); swslot = pg->uanon->an_swslot; } KASSERT(swslot); } #else /* defined(VMSWAP) */ panic("%s: swap", __func__); #endif /* defined(VMSWAP) */ } for (i = 0; i < npages; i++) { #if defined(VMSWAP) bool anon_disposed = false; /* XXX gcc */ #endif /* defined(VMSWAP) */ pg = pgs[i]; KASSERT(swap || pg->uobject == uobj); UVMHIST_LOG(ubchist, "pg %#jx", (uintptr_t)pg, 0,0,0); #if defined(VMSWAP) /* * for swap i/os, lock each page's object (or anon) * individually since each page may need a different lock. */ if (swap) { if (pg->uobject != NULL) { slock = pg->uobject->vmobjlock; } else { slock = pg->uanon->an_lock; } rw_enter(slock, RW_WRITER); anon_disposed = (pg->flags & PG_RELEASED) != 0; KASSERT(!anon_disposed || pg->uobject != NULL || pg->uanon->an_ref == 0); } #endif /* defined(VMSWAP) */ if (write && uobj != NULL) { KASSERT(uvm_obj_page_writeback_p(pg)); uvm_obj_page_clear_writeback(pg); } /* * process errors. for reads, just mark the page to be freed. * for writes, if the error was ENOMEM, we assume this was * a transient failure so we mark the page dirty so that * we'll try to write it again later. for all other write * errors, we assume the error is permanent, thus the data * in the page is lost. bummer. */ if (error) { int slot __unused; /* used for VMSWAP */ if (!write) { pg->flags |= PG_RELEASED; continue; } else if (error == ENOMEM) { if (pg->flags & PG_PAGEOUT) { pg->flags &= ~PG_PAGEOUT; pageout_done++; } uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); uvm_pagelock(pg); uvm_pageactivate(pg); uvm_pageunlock(pg); slot = 0; } else slot = SWSLOT_BAD; #if defined(VMSWAP) if (swap) { if (pg->uobject != NULL) { int oldslot __diagused; oldslot = uao_set_swslot(pg->uobject, pg->offset >> PAGE_SHIFT, slot); KASSERT(oldslot == swslot + i); } else { KASSERT(pg->uanon->an_swslot == swslot + i); pg->uanon->an_swslot = slot; } } #endif /* defined(VMSWAP) */ } /* * if the page is PG_FAKE, this must have been a read to * initialize the page. clear PG_FAKE and activate the page. */ if (pg->flags & PG_FAKE) { KASSERT(!write); pg->flags &= ~PG_FAKE; #if defined(READAHEAD_STATS) pg->flags |= PG_READAHEAD; uvm_ra_total.ev_count++; #endif /* defined(READAHEAD_STATS) */ KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN); uvm_pagelock(pg); uvm_pageenqueue(pg); uvm_pageunlock(pg); } #if defined(VMSWAP) /* * for swap pages, unlock everything for this page now. */ if (swap) { if (pg->uobject == NULL && anon_disposed) { uvm_anon_release(pg->uanon); } else { uvm_page_unbusy(&pg, 1); rw_exit(slock); } } #endif /* defined(VMSWAP) */ } if (pageout_done != 0) { uvm_pageout_done(pageout_done); } if (!swap) { uvm_page_unbusy(pgs, npages); rw_exit(slock); } else { #if defined(VMSWAP) KASSERT(write); /* these pages are now only in swap. */ if (error != ENOMEM) { atomic_add_int(&uvmexp.swpgonly, npages); } if (error) { if (error != ENOMEM) uvm_swap_markbad(swslot, npages); else uvm_swap_free(swslot, npages); } atomic_dec_uint(&uvmexp.pdpending); #endif /* defined(VMSWAP) */ } } /* * uvm_aio_aiodone: do iodone processing for async i/os. * this should be called in thread context, not interrupt context. */ void uvm_aio_aiodone(struct buf *bp) { const int npages = bp->b_bufsize >> PAGE_SHIFT; struct vm_page *pgs[howmany(MAXPHYS, MIN_PAGE_SIZE)]; int i, error; bool write; UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(ubchist, "bp %#jx", (uintptr_t)bp, 0,0,0); KASSERT(bp->b_bufsize <= MAXPHYS); KASSERT(npages <= __arraycount(pgs)); error = bp->b_error; write = (bp->b_flags & B_READ) == 0; for (i = 0; i < npages; i++) { pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT)); UVMHIST_LOG(ubchist, "pgs[%jd] = %#jx", i, (uintptr_t)pgs[i], 0, 0); } uvm_pagermapout((vaddr_t)bp->b_data, npages); uvm_aio_aiodone_pages(pgs, npages, write, error); if (write && (bp->b_cflags & BC_AGE) != 0) { mutex_enter(bp->b_objlock); vwakeup(bp); mutex_exit(bp->b_objlock); } putiobuf(bp); } /* * uvm_pageratop: convert KVAs in the pager map back to their page * structures. */ struct vm_page * uvm_pageratop(vaddr_t kva) { struct vm_page *pg; paddr_t pa; bool rv __diagused; rv = pmap_extract(pmap_kernel(), kva, &pa); KASSERT(rv); pg = PHYS_TO_VM_PAGE(pa); KASSERT(pg != NULL); return (pg); }