Lines Matching defs:pvo
305 #define PVO_WIRED 0x0010 /* PVO entry is wired */
306 #define PVO_MANAGED 0x0020 /* PVO e. for managed page */
307 #define PVO_EXECUTABLE 0x0040 /* PVO e. for executable page */
308 #define PVO_WIRED_P(pvo) ((pvo)->pvo_vaddr & PVO_WIRED)
309 #define PVO_MANAGED_P(pvo) ((pvo)->pvo_vaddr & PVO_MANAGED)
310 #define PVO_EXECUTABLE_P(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE)
311 #define PVO_ENTER_INSERT 0 /* PVO has been removed */
312 #define PVO_SPILL_UNSET 1 /* PVO has been evicted */
313 #define PVO_SPILL_SET 2 /* PVO has been spilled */
314 #define PVO_SPILL_INSERT 3 /* PVO has been inserted */
315 #define PVO_PMAP_PAGE_PROTECT 4 /* PVO has changed */
316 #define PVO_PMAP_PROTECT 5 /* PVO has changed */
317 #define PVO_REMOVE 6 /* PVO has been removed */
328 #define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF)
329 #define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
330 #define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
331 #define PVO_PTEGIDX_CLR(pvo) \
332 ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
333 #define PVO_PTEGIDX_SET(pvo,i) \
334 ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
335 #define PVO_WHERE(pvo,w) \
336 ((pvo)->pvo_vaddr &= ~(PVO_WHERE_MASK << PVO_WHERE_SHFT), \
337 (pvo)->pvo_vaddr |= ((PVO_ ## w) << PVO_WHERE_SHFT))
340 struct pvo_tqhead *pmap_pvo_table; /* pvo entries by ptegroup index */
343 struct pool pmap_pvo_pool; /* pool for pvo entries */
370 #define PMAP_PVO_CHECK(pvo) \
373 pmap_pvo_check(pvo); \
376 #define PMAP_PVO_CHECK(pvo) do { } while (/*CONSTCOND*/0)
884 struct pvo_entry *pvo, *source_pvo, *victim_pvo;
901 * Find source pvo.
905 TAILQ_FOREACH(pvo, spvoh, pvo_olink) {
907 * We need to find pvo entry for this address...
909 PMAP_PVO_CHECK(pvo); /* sanity check */
912 * If we haven't found the source and we come to a PVO with
916 if ((pvo->pvo_pte.pte_hi & PTE_VALID) != 0)
919 if (pm == pvo->pvo_pmap && addr == PVO_VADDR(pvo)) {
921 if (!PVO_EXECUTABLE_P(pvo))
925 PVO_VADDR(pvo) >> ADDR_SR_SHFT;
929 KASSERT(!PVO_PTEGIDX_ISSET(pvo));
931 source_pvo = pvo;
944 i = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
946 PVO_PTEGIDX_SET(pvo, i);
947 PMAP_PVO_CHECK(pvo); /* sanity check */
948 PVO_WHERE(pvo, SPILL_INSERT);
949 pvo->pvo_pmap->pm_evictions--;
951 PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) != 0
955 TAILQ_REMOVE(spvoh, pvo, pvo_olink);
956 TAILQ_INSERT_TAIL(spvoh, pvo, pvo_olink);
993 * We also need the pvo entry of the victim we are replacing
1001 TAILQ_FOREACH(pvo, vpvoh, pvo_olink) {
1002 PMAP_PVO_CHECK(pvo); /* sanity check */
1004 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0)
1007 if (pmap_pte_compare(pt, &pvo->pvo_pte)) {
1008 victim_pvo = pvo;
1013 panic("%s: victim p-pte (%p) has no pvo entry!",
1018 * The victim should be not be a kernel PVO/PTE entry.
1036 * To enforce the PVO list ordering constraint that all
1038 * move the source PVO to the tail of its list and the
1039 * victim PVO to the head of its list (which might not be
1283 pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx)
1291 pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo);
1292 if (pvo->pvo_pte.pte_hi & PTE_HID)
1298 pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
1303 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0)
1312 ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr);
1313 pteidx = pmap_pvo_pte_index(pvo, ptegidx);
1321 if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) {
1322 panic("pmap_pvo_to_pte: pvo %p: has valid pte in "
1323 "pvo but no valid pte index", pvo);
1325 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) {
1326 panic("pmap_pvo_to_pte: pvo %p: has valid pte index in "
1327 "pvo but no valid pte", pvo);
1330 if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) {
1331 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) {
1335 panic("pmap_pvo_to_pte: pvo %p: has valid pte in "
1336 "pmap_pteg_table %p but invalid in pvo",
1337 pvo, pt);
1339 if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) != 0) {
1343 panic("pmap_pvo_to_pte: pvo %p: pvo pte does "
1345 pvo, pt);
1350 if (pvo->pvo_pte.pte_hi & PTE_VALID) {
1354 panic("pmap_pvo_to_pte: pvo %p: has nomatching pte %p in "
1355 "pmap_pteg_table but valid in pvo", pvo, pt);
1364 struct pvo_entry *pvo;
1370 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
1372 if ((uintptr_t) pvo >= PMAP_DIRECT_MAPPED_LEN)
1373 panic("pmap_pvo_find_va: invalid pvo %p on "
1374 "list %#x (%p)", pvo, ptegidx,
1377 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
1379 *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx);
1380 return pvo;
1391 pmap_pvo_check(const struct pvo_entry *pvo)
1400 if ((uintptr_t)(pvo+1) >= PMAP_DIRECT_MAPPED_LEN)
1401 panic("pmap_pvo_check: pvo %p: invalid address", pvo);
1403 if ((uintptr_t)(pvo->pvo_pmap+1) >= PMAP_DIRECT_MAPPED_LEN) {
1404 printf("pmap_pvo_check: pvo %p: invalid pmap address %p\n",
1405 pvo, pvo->pvo_pmap);
1409 if ((uintptr_t)TAILQ_NEXT(pvo, pvo_olink) >= PMAP_DIRECT_MAPPED_LEN ||
1410 (((uintptr_t)TAILQ_NEXT(pvo, pvo_olink)) & 0x1f) != 0) {
1411 printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n",
1412 pvo, TAILQ_NEXT(pvo, pvo_olink));
1416 if ((uintptr_t)LIST_NEXT(pvo, pvo_vlink) >= PMAP_DIRECT_MAPPED_LEN ||
1417 (((uintptr_t)LIST_NEXT(pvo, pvo_vlink)) & 0x1f) != 0) {
1418 printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n",
1419 pvo, LIST_NEXT(pvo, pvo_vlink));
1423 if (PVO_MANAGED_P(pvo)) {
1424 pvo_head = pa_to_pvoh(pvo->pvo_pte.pte_lo & PTE_RPGN, NULL);
1426 if (pvo0 == pvo)
1430 printf("pmap_pvo_check: pvo %p: not present "
1431 "on its vlist head %p\n", pvo, pvo_head);
1435 KASSERT(pvo->pvo_vaddr >= VM_MIN_KERNEL_ADDRESS);
1436 if (__predict_false(pvo->pvo_vaddr < VM_MIN_KERNEL_ADDRESS))
1439 if (pvo != pmap_pvo_find_va(pvo->pvo_pmap, pvo->pvo_vaddr, NULL)) {
1440 printf("pmap_pvo_check: pvo %p: not present "
1441 "on its olist head\n", pvo);
1444 pt = pmap_pvo_to_pte(pvo, -1);
1446 if (pvo->pvo_pte.pte_hi & PTE_VALID) {
1447 printf("pmap_pvo_check: pvo %p: pte_hi VALID but "
1448 "no PTE\n", pvo);
1455 printf("pmap_pvo_check: pvo %p: pte %p not in "
1456 "pteg table\n", pvo, pt);
1459 if (((((uintptr_t) pt) >> 3) & 7) != PVO_PTEGIDX_GET(pvo)) {
1460 printf("pmap_pvo_check: pvo %p: pte_hi VALID but "
1461 "no PTE\n", pvo);
1464 if (pvo->pvo_pte.pte_hi != pt->pte_hi) {
1465 printf("pmap_pvo_check: pvo %p: pte_hi differ: "
1466 "%#" _PRIxpte "/%#" _PRIxpte "\n", pvo,
1467 pvo->pvo_pte.pte_hi,
1471 if (((pvo->pvo_pte.pte_lo ^ pt->pte_lo) &
1473 printf("pmap_pvo_check: pvo %p: pte_lo differ: "
1474 "%#" _PRIxpte "/%#" _PRIxpte "\n", pvo,
1475 (pvo->pvo_pte.pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)),
1479 if ((pmap_pte_to_va(pt) ^ PVO_VADDR(pvo)) & 0x0fffffff) {
1480 printf("pmap_pvo_check: pvo %p: PTE %p derived VA %#" _PRIxva ""
1481 " doesn't not match PVO's VA %#" _PRIxva "\n",
1482 pvo, pt, pmap_pte_to_va(pt), PVO_VADDR(pvo));
1489 panic("pmap_pvo_check: pvo %p, pm %p: bugcheck!", pvo,
1490 pvo->pvo_pmap);
1497 * Search the PVO table looking for a non-wired entry.
1505 struct pvo_entry *pvo;
1512 TAILQ_FOREACH(pvo, pvoh, pvo_olink) {
1513 if (!PVO_WIRED_P(pvo)) {
1514 pmap_pvo_remove(pvo, -1, NULL);
1517 return pvo;
1531 struct pvo_entry *pvo;
1555 * pvo entry if there a mapping.
1557 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
1558 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
1561 ((pvo->pvo_pte.pte_lo ^ (pa|pte_lo)) &
1564 printf("pmap_pvo_enter: pvo %p: dup %#" _PRIxpte "/%#" _PRIxpa "\n",
1565 pvo, pvo->pvo_pte.pte_lo, pte_lo|pa);
1567 pvo->pvo_pte.pte_hi,
1569 pmap_pte_print(pmap_pvo_to_pte(pvo, -1));
1576 pmap_pvo_remove(pvo, -1, NULL);
1588 if (pvo == NULL) {
1589 pvo = pool_get(pl, poolflags);
1591 KASSERT((vaddr_t)pvo < VM_MIN_KERNEL_ADDRESS);
1597 if (pvo && (flags & PMAP_CANFAIL) != 0 &&
1600 pool_put(pl, pvo);
1601 pvo = NULL;
1609 if (pvo == NULL) {
1610 pvo = pmap_pvo_reclaim();
1611 if (pvo == NULL) {
1623 pvo->pvo_vaddr = va;
1624 pvo->pvo_pmap = pm;
1625 pvo->pvo_vaddr &= ~ADDR_POFF;
1628 pvo_set_exec(pvo);
1631 pvo->pvo_vaddr |= PVO_WIRED;
1633 pvo->pvo_vaddr |= PVO_MANAGED;
1638 pmap_pte_create(&pvo->pvo_pte, pm, va, pa | pte_lo);
1641 LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
1642 if (PVO_WIRED_P(pvo))
1643 pvo->pvo_pmap->pm_stats.wired_count++;
1644 pvo->pvo_pmap->pm_stats.resident_count++;
1648 "pmap_pvo_enter: pvo %p: pm %p va %#" _PRIxva " pa %#" _PRIxpa "\n",
1649 pvo, pm, va, pa);
1656 i = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
1658 PVO_PTEGIDX_SET(pvo, i);
1659 PVO_WHERE(pvo, ENTER_INSERT);
1660 PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID)
1662 TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink);
1669 TAILQ_INSERT_HEAD(pvoh, pvo, pvo_olink);
1680 PMAP_PVO_CHECK(pvo); /* sanity check */
1689 pmap_pvo_remove(struct pvo_entry *pvo, int pteidx, struct pvo_head *pvol)
1703 ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr);
1704 pteidx = pmap_pvo_pte_index(pvo, ptegidx);
1707 if (pvo->pvo_pte.pte_hi & PTE_HID)
1710 PMAP_PVO_CHECK(pvo); /* sanity check */
1716 pt = pmap_pvo_to_pte(pvo, pteidx);
1718 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
1719 PVO_WHERE(pvo, REMOVE);
1720 PVO_PTEGIDX_CLR(pvo);
1723 KASSERT(pvo->pvo_pmap->pm_evictions > 0);
1724 pvo->pvo_pmap->pm_evictions--;
1730 if (PVO_EXECUTABLE_P(pvo))
1731 pvo_clear_exec(pvo);
1736 pvo->pvo_pmap->pm_stats.resident_count--;
1737 if (PVO_WIRED_P(pvo))
1738 pvo->pvo_pmap->pm_stats.wired_count--;
1743 * Remove the PVO from the P/V list.
1745 if (PVO_MANAGED_P(pvo)) {
1746 register_t ptelo = pvo->pvo_pte.pte_lo;
1775 LIST_REMOVE(pvo, pvo_vlink);
1782 * Remove the PVO from its list and return it to the pool.
1784 TAILQ_REMOVE(&pmap_pvo_table[ptegidx], pvo, pvo_olink);
1786 LIST_INSERT_HEAD(pvol, pvo, pvo_vlink);
1794 pmap_pvo_free(struct pvo_entry *pvo)
1797 pool_put(&pmap_pvo_pool, pvo);
1803 struct pvo_entry *pvo, *npvo;
1805 for (pvo = LIST_FIRST(pvol); pvo != NULL; pvo
1806 npvo = LIST_NEXT(pvo, pvo_vlink);
1807 LIST_REMOVE(pvo, pvo_vlink);
1808 pmap_pvo_free(pvo);
1818 pvo_set_exec(struct pvo_entry *pvo)
1820 struct pmap *pm = pvo->pvo_pmap;
1822 if (pm == pmap_kernel() || PVO_EXECUTABLE_P(pvo)) {
1825 pvo->pvo_vaddr |= PVO_EXECUTABLE;
1828 int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT;
1842 pvo_clear_exec(struct pvo_entry *pvo)
1844 struct pmap *pm = pvo->pvo_pmap;
1846 if (pm == pmap_kernel() || !PVO_EXECUTABLE_P(pvo)) {
1849 pvo->pvo_vaddr &= ~PVO_EXECUTABLE;
1852 int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT;
2052 struct pvo_entry *pvo;
2060 pvo = pmap_pvo_find_va(pm, va, &pteidx);
2061 if (pvo != NULL) {
2062 pmap_pvo_remove(pvo, pteidx, &pvol);
2136 struct pvo_entry *pvo;
2183 pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
2184 if (pvo != NULL) {
2185 PMAP_PVO_CHECK(pvo); /* sanity check */
2187 *pap = (pvo->pvo_pte.pte_lo & PTE_RPGN)
2192 return pvo != NULL;
2201 struct pvo_entry *pvo;
2225 pvo = pmap_pvo_find_va(pm, va, &pteidx);
2226 if (pvo == NULL)
2228 PMAP_PVO_CHECK(pvo); /* sanity check */
2234 pvo_clear_exec(pvo);
2241 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR)
2248 pt = pmap_pvo_to_pte(pvo, pteidx);
2252 pvo->pvo_pte.pte_lo &= ~PTE_PP;
2253 pvo->pvo_pte.pte_lo |= PTE_BR;
2256 * If the PVO is in the page table, update
2260 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
2261 PVO_WHERE(pvo, PMAP_PROTECT);
2265 PMAP_PVO_CHECK(pvo); /* sanity check */
2274 struct pvo_entry *pvo;
2279 pvo = pmap_pvo_find_va(pm, va, NULL);
2280 if (pvo != NULL) {
2281 if (PVO_WIRED_P(pvo)) {
2282 pvo->pvo_vaddr &= ~PVO_WIRED;
2285 PMAP_PVO_CHECK(pvo); /* sanity check */
2295 struct pvo_entry *pvo, *next_pvo;
2320 for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
2321 next_pvo = LIST_NEXT(pvo, pvo_vlink);
2322 PMAP_PVO_CHECK(pvo); /* sanity check */
2328 pmap_pvo_remove(pvo, -1, &pvol);
2334 * flag in the PVO.
2337 pvo_clear_exec(pvo);
2343 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) {
2344 PMAP_PVO_CHECK(pvo);
2353 pt = pmap_pvo_to_pte(pvo, -1);
2354 pvo->pvo_pte.pte_lo &= ~PTE_PP;
2355 pvo->pvo_pte.pte_lo |= PTE_BR;
2357 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
2358 PVO_WHERE(pvo, PMAP_PAGE_PROTECT);
2361 PMAP_PVO_CHECK(pvo); /* sanity check */
2434 struct pvo_entry *pvo;
2446 LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) {
2447 PMAP_PVO_CHECK(pvo); /* sanity check */
2452 if (pvo->pvo_pte.pte_lo & ptebit) {
2454 PMAP_PVO_CHECK(pvo); /* sanity check */
2466 LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) {
2467 PMAP_PVO_CHECK(pvo); /* sanity check */
2469 * See if this pvo have a valid PTE. If so, fetch the
2473 pt = pmap_pvo_to_pte(pvo, -1);
2475 pmap_pte_synch(pt, &pvo->pvo_pte);
2476 if (pvo->pvo_pte.pte_lo & ptebit) {
2478 PMAP_PVO_CHECK(pvo); /* sanity check */
2494 struct pvo_entry *pvo;
2514 * can reset the right ones). Note that since the pvo entries and
2522 * For each pvo entry, clear pvo's ptebit. If this pvo have a
2525 LIST_FOREACH(pvo, pvoh, pvo_vlink) {
2526 PMAP_PVO_CHECK(pvo); /* sanity check */
2527 pt = pmap_pvo_to_pte(pvo, -1);
2533 if ((pvo->pvo_pte.pte_lo & ptebit) == 0)
2534 pmap_pte_synch(pt, &pvo->pvo_pte);
2539 if (pvo->pvo_pte.pte_lo & ptebit)
2540 pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit);
2542 rv |= pvo->pvo_pte.pte_lo & (PTE_CHG|PTE_REF);
2543 pvo->pvo_pte.pte_lo &= ~ptebit;
2544 PMAP_PVO_CHECK(pvo); /* sanity check */
2575 struct pvo_entry *pvo;
2585 pvo = pmap_pvo_find_va(p->p_vmspace->vm_map.pmap, va, NULL);
2586 if (pvo != NULL && PVO_EXECUTABLE_P(pvo)) {
2588 (pvo->pvo_pte.pte_lo & PTE_RPGN) | offset, seglen);
2589 PMAP_PVO_CHECK(pvo);
2771 struct pvo_entry *pvo;
2775 pvo = pmap_pvo_find_va(pm, va, &pteidx);
2776 if (pvo != NULL) {
2777 pt = pmap_pvo_to_pte(pvo, pteidx);
2794 struct pvo_entry *pvo;
2803 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
2835 struct pvo_entry *pvo;
2836 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
2837 if ((uintptr_t) pvo >= PMAP_DIRECT_MAPPED_LEN)
2838 panic("pmap_pvo_verify: invalid pvo %p "
2839 "on list %#x", pvo, ptegidx);
2840 pmap_pvo_check(pvo);
Indexes created Thu Oct 02 10:09:58 GMT 2025