/* $NetBSD: uvm_bio.c,v 1.128 2023/04/09 09:00:56 riastradh Exp $ */ /* * Copyright (c) 1998 Chuck Silvers. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * uvm_bio.c: buffered i/o object mapping cache */ #include __KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.128 2023/04/09 09:00:56 riastradh Exp $"); #include "opt_uvmhist.h" #include "opt_ubc.h" #include #include #include #include #include #include #include #include /* for ilog2() */ #include #include #ifdef PMAP_DIRECT # define UBC_USE_PMAP_DIRECT #endif /* * local functions */ static int ubc_fault(struct uvm_faultinfo *, vaddr_t, struct vm_page **, int, int, vm_prot_t, int); static struct ubc_map *ubc_find_mapping(struct uvm_object *, voff_t); static int ubchash_stats(struct hashstat_sysctl *hs, bool fill); #ifdef UBC_USE_PMAP_DIRECT static int __noinline ubc_uiomove_direct(struct uvm_object *, struct uio *, vsize_t, int, int); static void __noinline ubc_zerorange_direct(struct uvm_object *, off_t, size_t, int); /* XXX disabled by default until the kinks are worked out. */ bool ubc_direct = false; #endif /* * local data structures */ #define UBC_HASH(uobj, offset) \ (((((u_long)(uobj)) >> 8) + (((u_long)(offset)) >> PAGE_SHIFT)) & \ ubc_object.hashmask) #define UBC_QUEUE(offset) \ (&ubc_object.inactive[(((u_long)(offset)) >> ubc_winshift) & \ (UBC_NQUEUES - 1)]) #define UBC_UMAP_ADDR(u) \ (vaddr_t)(ubc_object.kva + (((u) - ubc_object.umap) << ubc_winshift)) #define UMAP_PAGES_LOCKED 0x0001 #define UMAP_MAPPING_CACHED 0x0002 struct ubc_map { struct uvm_object * uobj; /* mapped object */ voff_t offset; /* offset into uobj */ voff_t writeoff; /* write offset */ vsize_t writelen; /* write len */ int refcount; /* refcount on mapping */ int flags; /* extra state */ int advice; LIST_ENTRY(ubc_map) hash; /* hash table */ TAILQ_ENTRY(ubc_map) inactive; /* inactive queue */ LIST_ENTRY(ubc_map) list; /* per-object list */ }; TAILQ_HEAD(ubc_inactive_head, ubc_map); static struct ubc_object { struct uvm_object uobj; /* glue for uvm_map() */ char *kva; /* where ubc_object is mapped */ struct ubc_map *umap; /* array of ubc_map's */ LIST_HEAD(, ubc_map) *hash; /* hashtable for cached ubc_map's */ u_long hashmask; /* mask for hashtable */ struct ubc_inactive_head *inactive; /* inactive queues for ubc_map's */ } ubc_object; const struct uvm_pagerops ubc_pager = { .pgo_fault = ubc_fault, /* ... rest are NULL */ }; /* Use value at least as big as maximum page size supported by architecture */ #define UBC_MAX_WINSHIFT \ ((1 << UBC_WINSHIFT) > MAX_PAGE_SIZE ? UBC_WINSHIFT : ilog2(MAX_PAGE_SIZE)) int ubc_nwins = UBC_NWINS; const int ubc_winshift = UBC_MAX_WINSHIFT; const int ubc_winsize = 1 << UBC_MAX_WINSHIFT; #if defined(PMAP_PREFER) int ubc_nqueues; #define UBC_NQUEUES ubc_nqueues #else #define UBC_NQUEUES 1 #endif #if defined(UBC_STATS) #define UBC_EVCNT_DEFINE(name) \ struct evcnt ubc_evcnt_##name = \ EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "ubc", #name); \ EVCNT_ATTACH_STATIC(ubc_evcnt_##name); #define UBC_EVCNT_INCR(name) ubc_evcnt_##name.ev_count++ #else /* defined(UBC_STATS) */ #define UBC_EVCNT_DEFINE(name) /* nothing */ #define UBC_EVCNT_INCR(name) /* nothing */ #endif /* defined(UBC_STATS) */ UBC_EVCNT_DEFINE(wincachehit) UBC_EVCNT_DEFINE(wincachemiss) UBC_EVCNT_DEFINE(faultbusy) /* * ubc_init * * init pager private data structures. */ void ubc_init(void) { /* * Make sure ubc_winshift is sane. */ KASSERT(ubc_winshift >= PAGE_SHIFT); /* * init ubc_object. * alloc and init ubc_map's. * init inactive queues. * alloc and init hashtable. * map in ubc_object. */ uvm_obj_init(&ubc_object.uobj, &ubc_pager, true, UVM_OBJ_KERN); ubc_object.umap = kmem_zalloc(ubc_nwins * sizeof(struct ubc_map), KM_SLEEP); if (ubc_object.umap == NULL) panic("ubc_init: failed to allocate ubc_map"); vaddr_t va = (vaddr_t)1L; #ifdef PMAP_PREFER PMAP_PREFER(0, &va, 0, 0); /* kernel is never topdown */ ubc_nqueues = va >> ubc_winshift; if (ubc_nqueues == 0) { ubc_nqueues = 1; } #endif ubc_object.inactive = kmem_alloc(UBC_NQUEUES * sizeof(struct ubc_inactive_head), KM_SLEEP); for (int i = 0; i < UBC_NQUEUES; i++) { TAILQ_INIT(&ubc_object.inactive[i]); } for (int i = 0; i < ubc_nwins; i++) { struct ubc_map *umap; umap = &ubc_object.umap[i]; TAILQ_INSERT_TAIL(&ubc_object.inactive[i & (UBC_NQUEUES - 1)], umap, inactive); } ubc_object.hash = hashinit(ubc_nwins, HASH_LIST, true, &ubc_object.hashmask); for (int i = 0; i <= ubc_object.hashmask; i++) { LIST_INIT(&ubc_object.hash[i]); } if (uvm_map(kernel_map, (vaddr_t *)&ubc_object.kva, ubc_nwins << ubc_winshift, &ubc_object.uobj, 0, (vsize_t)va, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_RANDOM, UVM_FLAG_NOMERGE)) != 0) { panic("ubc_init: failed to map ubc_object"); } hashstat_register("ubchash", ubchash_stats); } void ubchist_init(void) { UVMHIST_INIT(ubchist, 300); } /* * ubc_fault_page: helper of ubc_fault to handle a single page. * * => Caller has UVM object locked. * => Caller will perform pmap_update(). */ static inline int ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap, struct vm_page *pg, vm_prot_t prot, vm_prot_t access_type, vaddr_t va) { vm_prot_t mask; int error; bool rdonly; KASSERT(rw_write_held(pg->uobject->vmobjlock)); KASSERT((pg->flags & PG_FAKE) == 0); if (pg->flags & PG_RELEASED) { uvm_pagefree(pg); return 0; } if (pg->loan_count != 0) { /* * Avoid unneeded loan break, if possible. */ if ((access_type & VM_PROT_WRITE) == 0) { prot &= ~VM_PROT_WRITE; } if (prot & VM_PROT_WRITE) { struct vm_page *newpg; newpg = uvm_loanbreak(pg); if (newpg == NULL) { uvm_page_unbusy(&pg, 1); return ENOMEM; } pg = newpg; } } /* * Note that a page whose backing store is partially allocated * is marked as PG_RDONLY. * * it's a responsibility of ubc_alloc's caller to allocate backing * blocks before writing to the window. */ KASSERT((pg->flags & PG_RDONLY) == 0 || (access_type & VM_PROT_WRITE) == 0 || pg->offset < umap->writeoff || pg->offset + PAGE_SIZE > umap->writeoff + umap->writelen); rdonly = uvm_pagereadonly_p(pg); mask = rdonly ? ~VM_PROT_WRITE : VM_PROT_ALL; error = pmap_enter(ufi->orig_map->pmap, va, VM_PAGE_TO_PHYS(pg), prot & mask, PMAP_CANFAIL | (access_type & mask)); uvm_pagelock(pg); uvm_pageactivate(pg); uvm_pagewakeup(pg); uvm_pageunlock(pg); pg->flags &= ~PG_BUSY; UVM_PAGE_OWN(pg, NULL); return error; } /* * ubc_fault: fault routine for ubc mapping */ static int ubc_fault(struct uvm_faultinfo *ufi, vaddr_t ign1, struct vm_page **ign2, int ign3, int ign4, vm_prot_t access_type, int flags) { struct uvm_object *uobj; struct ubc_map *umap; vaddr_t va, eva, ubc_offset, slot_offset; struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)]; int i, error, npages; vm_prot_t prot; UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); /* * no need to try with PGO_LOCKED... * we don't need to have the map locked since we know that * no one will mess with it until our reference is released. */ if (flags & PGO_LOCKED) { uvmfault_unlockall(ufi, NULL, &ubc_object.uobj); flags &= ~PGO_LOCKED; } va = ufi->orig_rvaddr; ubc_offset = va - (vaddr_t)ubc_object.kva; umap = &ubc_object.umap[ubc_offset >> ubc_winshift]; KASSERT(umap->refcount != 0); KASSERT((umap->flags & UMAP_PAGES_LOCKED) == 0); slot_offset = ubc_offset & (ubc_winsize - 1); /* * some platforms cannot write to individual bytes atomically, so * software has to do read/modify/write of larger quantities instead. * this means that the access_type for "write" operations * can be VM_PROT_READ, which confuses us mightily. * * deal with this by resetting access_type based on the info * that ubc_alloc() stores for us. */ access_type = umap->writelen ? VM_PROT_WRITE : VM_PROT_READ; UVMHIST_LOG(ubchist, "va %#jx ubc_offset %#jx access_type %jd", va, ubc_offset, access_type, 0); if ((access_type & VM_PROT_WRITE) != 0) { #ifndef PRIxOFF /* XXX */ #define PRIxOFF "jx" /* XXX */ #endif /* XXX */ KASSERTMSG((trunc_page(umap->writeoff) <= slot_offset), "out of range write: slot=%#"PRIxVSIZE" off=%#"PRIxOFF, slot_offset, (intmax_t)umap->writeoff); KASSERTMSG((slot_offset < umap->writeoff + umap->writelen), "out of range write: slot=%#"PRIxVADDR " off=%#"PRIxOFF" len=%#"PRIxVSIZE, slot_offset, (intmax_t)umap->writeoff, umap->writelen); } /* no umap locking needed since we have a ref on the umap */ uobj = umap->uobj; if ((access_type & VM_PROT_WRITE) == 0) { npages = (ubc_winsize - slot_offset) >> PAGE_SHIFT; } else { npages = (round_page(umap->offset + umap->writeoff + umap->writelen) - (umap->offset + slot_offset)) >> PAGE_SHIFT; flags |= PGO_PASTEOF; } again: memset(pgs, 0, sizeof (pgs)); rw_enter(uobj->vmobjlock, RW_WRITER); UVMHIST_LOG(ubchist, "slot_offset %#jx writeoff %#jx writelen %#jx ", slot_offset, umap->writeoff, umap->writelen, 0); UVMHIST_LOG(ubchist, "getpages uobj %#jx offset %#jx npages %jd", (uintptr_t)uobj, umap->offset + slot_offset, npages, 0); error = (*uobj->pgops->pgo_get)(uobj, umap->offset + slot_offset, pgs, &npages, 0, access_type, umap->advice, flags | PGO_NOBLOCKALLOC | PGO_NOTIMESTAMP); UVMHIST_LOG(ubchist, "getpages error %jd npages %jd", error, npages, 0, 0); if (error == EAGAIN) { kpause("ubc_fault", false, hz >> 2, NULL); goto again; } if (error) { return error; } /* * For virtually-indexed, virtually-tagged caches we should avoid * creating writable mappings when we do not absolutely need them, * since the "compatible alias" trick does not work on such caches. * Otherwise, we can always map the pages writable. */ #ifdef PMAP_CACHE_VIVT prot = VM_PROT_READ | access_type; #else prot = VM_PROT_READ | VM_PROT_WRITE; #endif va = ufi->orig_rvaddr; eva = ufi->orig_rvaddr + (npages << PAGE_SHIFT); UVMHIST_LOG(ubchist, "va %#jx eva %#jx", va, eva, 0, 0); /* * Note: normally all returned pages would have the same UVM object. * However, layered file-systems and e.g. tmpfs, may return pages * which belong to underlying UVM object. In such case, lock is * shared amongst the objects. */ rw_enter(uobj->vmobjlock, RW_WRITER); for (i = 0; va < eva; i++, va += PAGE_SIZE) { struct vm_page *pg; UVMHIST_LOG(ubchist, "pgs[%jd] = %#jx", i, (uintptr_t)pgs[i], 0, 0); pg = pgs[i]; if (pg == NULL || pg == PGO_DONTCARE) { continue; } KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock); error = ubc_fault_page(ufi, umap, pg, prot, access_type, va); if (error) { /* * Flush (there might be pages entered), drop the lock, * and perform uvm_wait(). Note: page will re-fault. */ pmap_update(ufi->orig_map->pmap); rw_exit(uobj->vmobjlock); uvm_wait("ubc_fault"); rw_enter(uobj->vmobjlock, RW_WRITER); } } /* Must make VA visible before the unlock. */ pmap_update(ufi->orig_map->pmap); rw_exit(uobj->vmobjlock); return 0; } /* * local functions */ static struct ubc_map * ubc_find_mapping(struct uvm_object *uobj, voff_t offset) { struct ubc_map *umap; LIST_FOREACH(umap, &ubc_object.hash[UBC_HASH(uobj, offset)], hash) { if (umap->uobj == uobj && umap->offset == offset) { return umap; } } return NULL; } /* * ubc interface functions */ /* * ubc_alloc: allocate a file mapping window */ static void * __noinline ubc_alloc(struct uvm_object *uobj, voff_t offset, vsize_t *lenp, int advice, int flags, struct vm_page **pgs, int *npagesp) { vaddr_t slot_offset, va; struct ubc_map *umap; voff_t umap_offset; int error; UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(ubchist, "uobj %#jx offset %#jx len %#jx", (uintptr_t)uobj, offset, *lenp, 0); KASSERT(*lenp > 0); umap_offset = (offset & ~((voff_t)ubc_winsize - 1)); slot_offset = (vaddr_t)(offset & ((voff_t)ubc_winsize - 1)); *lenp = MIN(*lenp, ubc_winsize - slot_offset); KASSERT(*lenp > 0); rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER); again: /* * The UVM object is already referenced. * Lock order: UBC object -> ubc_map::uobj. */ umap = ubc_find_mapping(uobj, umap_offset); if (umap == NULL) { struct uvm_object *oobj; UBC_EVCNT_INCR(wincachemiss); umap = TAILQ_FIRST(UBC_QUEUE(offset)); if (umap == NULL) { rw_exit(ubc_object.uobj.vmobjlock); kpause("ubc_alloc", false, hz >> 2, NULL); rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER); goto again; } va = UBC_UMAP_ADDR(umap); oobj = umap->uobj; /* * Remove from old hash (if any), add to new hash. */ if (oobj != NULL) { /* * Mapping must be removed before the list entry, * since there is a race with ubc_purge(). */ if (umap->flags & UMAP_MAPPING_CACHED) { umap->flags &= ~UMAP_MAPPING_CACHED; rw_enter(oobj->vmobjlock, RW_WRITER); pmap_remove(pmap_kernel(), va, va + ubc_winsize); pmap_update(pmap_kernel()); rw_exit(oobj->vmobjlock); } LIST_REMOVE(umap, hash); LIST_REMOVE(umap, list); } else { KASSERT((umap->flags & UMAP_MAPPING_CACHED) == 0); } umap->uobj = uobj; umap->offset = umap_offset; LIST_INSERT_HEAD(&ubc_object.hash[UBC_HASH(uobj, umap_offset)], umap, hash); LIST_INSERT_HEAD(&uobj->uo_ubc, umap, list); } else { UBC_EVCNT_INCR(wincachehit); va = UBC_UMAP_ADDR(umap); } if (umap->refcount == 0) { TAILQ_REMOVE(UBC_QUEUE(offset), umap, inactive); } if (flags & UBC_WRITE) { KASSERTMSG(umap->writeoff == 0, "ubc_alloc: concurrent writes to uobj %p", uobj); KASSERTMSG(umap->writelen == 0, "ubc_alloc: concurrent writes to uobj %p", uobj); umap->writeoff = slot_offset; umap->writelen = *lenp; } umap->refcount++; umap->advice = advice; rw_exit(ubc_object.uobj.vmobjlock); UVMHIST_LOG(ubchist, "umap %#jx refs %jd va %#jx flags %#jx", (uintptr_t)umap, umap->refcount, (uintptr_t)va, flags); if (flags & UBC_FAULTBUSY) { int npages = (*lenp + (offset & (PAGE_SIZE - 1)) + PAGE_SIZE - 1) >> PAGE_SHIFT; int gpflags = PGO_SYNCIO|PGO_OVERWRITE|PGO_PASTEOF|PGO_NOBLOCKALLOC| PGO_NOTIMESTAMP; int i; KDASSERT(flags & UBC_WRITE); KASSERT(npages <= *npagesp); KASSERT(umap->refcount == 1); UBC_EVCNT_INCR(faultbusy); again_faultbusy: rw_enter(uobj->vmobjlock, RW_WRITER); if (umap->flags & UMAP_MAPPING_CACHED) { umap->flags &= ~UMAP_MAPPING_CACHED; pmap_remove(pmap_kernel(), va, va + ubc_winsize); } memset(pgs, 0, *npagesp * sizeof(pgs[0])); error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs, &npages, 0, VM_PROT_READ | VM_PROT_WRITE, advice, gpflags); UVMHIST_LOG(ubchist, "faultbusy getpages %jd", error, 0, 0, 0); if (error) { /* * Flush: the mapping above might have been removed. */ pmap_update(pmap_kernel()); goto out; } for (i = 0; i < npages; i++) { struct vm_page *pg = pgs[i]; KASSERT(pg->uobject == uobj); if (pg->loan_count != 0) { rw_enter(uobj->vmobjlock, RW_WRITER); if (pg->loan_count != 0) { pg = uvm_loanbreak(pg); } if (pg == NULL) { pmap_kremove(va, ubc_winsize); pmap_update(pmap_kernel()); uvm_page_unbusy(pgs, npages); rw_exit(uobj->vmobjlock); uvm_wait("ubc_alloc"); goto again_faultbusy; } rw_exit(uobj->vmobjlock); pgs[i] = pg; } pmap_kenter_pa( va + trunc_page(slot_offset) + (i << PAGE_SHIFT), VM_PAGE_TO_PHYS(pg), VM_PROT_READ | VM_PROT_WRITE, 0); } pmap_update(pmap_kernel()); umap->flags |= UMAP_PAGES_LOCKED; *npagesp = npages; } else { KASSERT((umap->flags & UMAP_PAGES_LOCKED) == 0); } out: return (void *)(va + slot_offset); } /* * ubc_release: free a file mapping window. */ static void __noinline ubc_release(void *va, int flags, struct vm_page **pgs, int npages) { struct ubc_map *umap; struct uvm_object *uobj; vaddr_t umapva; bool unmapped; UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(ubchist, "va %#jx", (uintptr_t)va, 0, 0, 0); umap = &ubc_object.umap[((char *)va - ubc_object.kva) >> ubc_winshift]; umapva = UBC_UMAP_ADDR(umap); uobj = umap->uobj; KASSERT(uobj != NULL); if (umap->flags & UMAP_PAGES_LOCKED) { const voff_t endoff = umap->writeoff + umap->writelen; const voff_t zerolen = round_page(endoff) - endoff; KASSERT(npages == (round_page(endoff) - trunc_page(umap->writeoff)) >> PAGE_SHIFT); KASSERT((umap->flags & UMAP_MAPPING_CACHED) == 0); if (zerolen) { memset((char *)umapva + endoff, 0, zerolen); } umap->flags &= ~UMAP_PAGES_LOCKED; rw_enter(uobj->vmobjlock, RW_WRITER); for (u_int i = 0; i < npages; i++) { struct vm_page *pg = pgs[i]; #ifdef DIAGNOSTIC paddr_t pa; bool rv; rv = pmap_extract(pmap_kernel(), umapva + umap->writeoff + (i << PAGE_SHIFT), &pa); KASSERT(rv); KASSERT(PHYS_TO_VM_PAGE(pa) == pg); #endif pg->flags &= ~PG_FAKE; KASSERTMSG(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY, "page %p not dirty", pg); KASSERT(pg->loan_count == 0); if (uvmpdpol_pageactivate_p(pg)) { uvm_pagelock(pg); uvm_pageactivate(pg); uvm_pageunlock(pg); } } pmap_kremove(umapva, ubc_winsize); pmap_update(pmap_kernel()); uvm_page_unbusy(pgs, npages); rw_exit(uobj->vmobjlock); unmapped = true; } else { unmapped = false; } rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER); umap->writeoff = 0; umap->writelen = 0; umap->refcount--; if (umap->refcount == 0) { if (flags & UBC_UNMAP) { /* * Invalidate any cached mappings if requested. * This is typically used to avoid leaving * incompatible cache aliases around indefinitely. */ rw_enter(uobj->vmobjlock, RW_WRITER); pmap_remove(pmap_kernel(), umapva, umapva + ubc_winsize); pmap_update(pmap_kernel()); rw_exit(uobj->vmobjlock); umap->flags &= ~UMAP_MAPPING_CACHED; LIST_REMOVE(umap, hash); LIST_REMOVE(umap, list); umap->uobj = NULL; TAILQ_INSERT_HEAD(UBC_QUEUE(umap->offset), umap, inactive); } else { if (!unmapped) { umap->flags |= UMAP_MAPPING_CACHED; } TAILQ_INSERT_TAIL(UBC_QUEUE(umap->offset), umap, inactive); } } UVMHIST_LOG(ubchist, "umap %#jx refs %jd", (uintptr_t)umap, umap->refcount, 0, 0); rw_exit(ubc_object.uobj.vmobjlock); } /* * ubc_uiomove: move data to/from an object. */ int ubc_uiomove(struct uvm_object *uobj, struct uio *uio, vsize_t todo, int advice, int flags) { const bool overwrite = (flags & UBC_FAULTBUSY) != 0; struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)]; voff_t off; int error, npages; KASSERT(todo <= uio->uio_resid); KASSERT(((flags & UBC_WRITE) != 0 && uio->uio_rw == UIO_WRITE) || ((flags & UBC_READ) != 0 && uio->uio_rw == UIO_READ)); #ifdef UBC_USE_PMAP_DIRECT /* * during direct access pages need to be held busy to prevent them * changing identity, and therefore if we read or write an object * into a mapped view of same we could deadlock while faulting. * * avoid the problem by disallowing direct access if the object * might be visible somewhere via mmap(). * * XXX concurrent reads cause thundering herd issues with PG_BUSY. * In the future enable by default for writes or if ncpu<=2, and * make the toggle override that. */ if ((ubc_direct && (flags & UBC_ISMAPPED) == 0) || (flags & UBC_FAULTBUSY) != 0) { return ubc_uiomove_direct(uobj, uio, todo, advice, flags); } #endif off = uio->uio_offset; error = 0; while (todo > 0) { vsize_t bytelen = todo; void *win; npages = __arraycount(pgs); win = ubc_alloc(uobj, off, &bytelen, advice, flags, pgs, &npages); if (error == 0) { error = uiomove(win, bytelen, uio); } if (error != 0 && overwrite) { /* * if we haven't initialized the pages yet, * do it now. it's safe to use memset here * because we just mapped the pages above. */ memset(win, 0, bytelen); } ubc_release(win, flags, pgs, npages); off += bytelen; todo -= bytelen; if (error != 0 && (flags & UBC_PARTIALOK) != 0) { break; } } return error; } /* * ubc_zerorange: set a range of bytes in an object to zero. */ void ubc_zerorange(struct uvm_object *uobj, off_t off, size_t len, int flags) { struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)]; int npages; #ifdef UBC_USE_PMAP_DIRECT if (ubc_direct || (flags & UBC_FAULTBUSY) != 0) { ubc_zerorange_direct(uobj, off, len, flags); return; } #endif /* * XXXUBC invent kzero() and use it */ while (len) { void *win; vsize_t bytelen = len; npages = __arraycount(pgs); win = ubc_alloc(uobj, off, &bytelen, UVM_ADV_NORMAL, UBC_WRITE, pgs, &npages); memset(win, 0, bytelen); ubc_release(win, flags, pgs, npages); off += bytelen; len -= bytelen; } } #ifdef UBC_USE_PMAP_DIRECT /* Copy data using direct map */ /* * ubc_alloc_direct: allocate a file mapping window using direct map */ static int __noinline ubc_alloc_direct(struct uvm_object *uobj, voff_t offset, vsize_t *lenp, int advice, int flags, struct vm_page **pgs, int *npages) { voff_t pgoff; int error; int gpflags = flags | PGO_NOTIMESTAMP | PGO_SYNCIO; int access_type = VM_PROT_READ; UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); if (flags & UBC_WRITE) { if (flags & UBC_FAULTBUSY) gpflags |= PGO_OVERWRITE | PGO_NOBLOCKALLOC; #if 0 KASSERT(!UVM_OBJ_NEEDS_WRITEFAULT(uobj)); #endif /* * Tell genfs_getpages() we already have the journal lock, * allow allocation past current EOF. */ gpflags |= PGO_JOURNALLOCKED | PGO_PASTEOF; access_type |= VM_PROT_WRITE; } else { /* Don't need the empty blocks allocated, PG_RDONLY is okay */ gpflags |= PGO_NOBLOCKALLOC; } pgoff = (offset & PAGE_MASK); *lenp = MIN(*lenp, ubc_winsize - pgoff); again: *npages = (*lenp + pgoff + PAGE_SIZE - 1) >> PAGE_SHIFT; KASSERT((*npages * PAGE_SIZE) <= ubc_winsize); KASSERT(*lenp + pgoff <= ubc_winsize); memset(pgs, 0, *npages * sizeof(pgs[0])); rw_enter(uobj->vmobjlock, RW_WRITER); error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs, npages, 0, access_type, advice, gpflags); UVMHIST_LOG(ubchist, "alloc_direct getpages %jd", error, 0, 0, 0); if (error) { if (error == EAGAIN) { kpause("ubc_alloc_directg", false, hz >> 2, NULL); goto again; } return error; } rw_enter(uobj->vmobjlock, RW_WRITER); for (int i = 0; i < *npages; i++) { struct vm_page *pg = pgs[i]; KASSERT(pg != NULL); KASSERT(pg != PGO_DONTCARE); KASSERT((pg->flags & PG_FAKE) == 0 || (gpflags & PGO_OVERWRITE)); KASSERT(pg->uobject->vmobjlock == uobj->vmobjlock); /* Avoid breaking loan if possible, only do it on write */ if ((flags & UBC_WRITE) && pg->loan_count != 0) { pg = uvm_loanbreak(pg); if (pg == NULL) { uvm_page_unbusy(pgs, *npages); rw_exit(uobj->vmobjlock); uvm_wait("ubc_alloc_directl"); goto again; } pgs[i] = pg; } /* Page must be writable by now */ KASSERT((pg->flags & PG_RDONLY) == 0 || (flags & UBC_WRITE) == 0); /* * XXX For aobj pages. No managed mapping - mark the page * dirty. */ if ((flags & UBC_WRITE) != 0) { uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); } } rw_exit(uobj->vmobjlock); return 0; } static void __noinline ubc_direct_release(struct uvm_object *uobj, int flags, struct vm_page **pgs, int npages) { rw_enter(uobj->vmobjlock, RW_WRITER); for (int i = 0; i < npages; i++) { struct vm_page *pg = pgs[i]; pg->flags &= ~PG_BUSY; UVM_PAGE_OWN(pg, NULL); if (pg->flags & PG_RELEASED) { pg->flags &= ~PG_RELEASED; uvm_pagefree(pg); continue; } if (uvm_pagewanted_p(pg) || uvmpdpol_pageactivate_p(pg)) { uvm_pagelock(pg); uvm_pageactivate(pg); uvm_pagewakeup(pg); uvm_pageunlock(pg); } /* Page was changed, no longer fake and neither clean. */ if (flags & UBC_WRITE) { KASSERTMSG(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY, "page %p not dirty", pg); pg->flags &= ~PG_FAKE; } } rw_exit(uobj->vmobjlock); } static int ubc_uiomove_process(void *win, size_t len, void *arg) { struct uio *uio = (struct uio *)arg; return uiomove(win, len, uio); } static int ubc_zerorange_process(void *win, size_t len, void *arg) { memset(win, 0, len); return 0; } static int __noinline ubc_uiomove_direct(struct uvm_object *uobj, struct uio *uio, vsize_t todo, int advice, int flags) { const bool overwrite = (flags & UBC_FAULTBUSY) != 0; voff_t off; int error, npages; struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)]; KASSERT(todo <= uio->uio_resid); KASSERT(((flags & UBC_WRITE) != 0 && uio->uio_rw == UIO_WRITE) || ((flags & UBC_READ) != 0 && uio->uio_rw == UIO_READ)); off = uio->uio_offset; error = 0; while (todo > 0) { vsize_t bytelen = todo; error = ubc_alloc_direct(uobj, off, &bytelen, advice, flags, pgs, &npages); if (error != 0) { /* can't do anything, failed to get the pages */ break; } if (error == 0) { error = uvm_direct_process(pgs, npages, off, bytelen, ubc_uiomove_process, uio); } if (overwrite) { voff_t endoff; /* * if we haven't initialized the pages yet due to an * error above, do it now. */ if (error != 0) { (void) uvm_direct_process(pgs, npages, off, bytelen, ubc_zerorange_process, NULL); } off += bytelen; todo -= bytelen; endoff = off & (PAGE_SIZE - 1); /* * zero out the remaining portion of the final page * (if any). */ if (todo == 0 && endoff != 0) { vsize_t zlen = PAGE_SIZE - endoff; (void) uvm_direct_process(pgs + npages - 1, 1, off, zlen, ubc_zerorange_process, NULL); } } else { off += bytelen; todo -= bytelen; } ubc_direct_release(uobj, flags, pgs, npages); if (error != 0 && ISSET(flags, UBC_PARTIALOK)) { break; } } return error; } static void __noinline ubc_zerorange_direct(struct uvm_object *uobj, off_t off, size_t todo, int flags) { int error, npages; struct vm_page *pgs[howmany(ubc_winsize, MIN_PAGE_SIZE)]; flags |= UBC_WRITE; error = 0; while (todo > 0) { vsize_t bytelen = todo; error = ubc_alloc_direct(uobj, off, &bytelen, UVM_ADV_NORMAL, flags, pgs, &npages); if (error != 0) { /* can't do anything, failed to get the pages */ break; } error = uvm_direct_process(pgs, npages, off, bytelen, ubc_zerorange_process, NULL); ubc_direct_release(uobj, flags, pgs, npages); off += bytelen; todo -= bytelen; } } #endif /* UBC_USE_PMAP_DIRECT */ /* * ubc_purge: disassociate ubc_map structures from an empty uvm_object. */ void ubc_purge(struct uvm_object *uobj) { struct ubc_map *umap; vaddr_t va; KASSERT(uobj->uo_npages == 0); /* * Safe to check without lock held, as ubc_alloc() removes * the mapping and list entry in the correct order. */ if (__predict_true(LIST_EMPTY(&uobj->uo_ubc))) { return; } rw_enter(ubc_object.uobj.vmobjlock, RW_WRITER); while ((umap = LIST_FIRST(&uobj->uo_ubc)) != NULL) { KASSERT(umap->refcount == 0); for (va = 0; va < ubc_winsize; va += PAGE_SIZE) { KASSERT(!pmap_extract(pmap_kernel(), va + UBC_UMAP_ADDR(umap), NULL)); } LIST_REMOVE(umap, list); LIST_REMOVE(umap, hash); umap->flags &= ~UMAP_MAPPING_CACHED; umap->uobj = NULL; } rw_exit(ubc_object.uobj.vmobjlock); } static int ubchash_stats(struct hashstat_sysctl *hs, bool fill) { struct ubc_map *umap; uint64_t chain; strlcpy(hs->hash_name, "ubchash", sizeof(hs->hash_name)); strlcpy(hs->hash_desc, "ubc object hash", sizeof(hs->hash_desc)); if (!fill) return 0; hs->hash_size = ubc_object.hashmask + 1; for (size_t i = 0; i < hs->hash_size; i++) { chain = 0; rw_enter(ubc_object.uobj.vmobjlock, RW_READER); LIST_FOREACH(umap, &ubc_object.hash[i], hash) { chain++; } rw_exit(ubc_object.uobj.vmobjlock); if (chain > 0) { hs->hash_used++; hs->hash_items += chain; if (chain > hs->hash_maxchain) hs->hash_maxchain = chain; } preempt_point(); } return 0; }