Lines Matching refs:hdr
187 * arc_buf_hdr_t is cached. If the hdr is cached and already has an arc_buf_t,
189 * bcopied from the existing arc_buf_t. If the hdr is cached but does not
195 * required when the hdr is caching uncompressed blocks or the compressed
198 * The diagram below shows an example of an uncompressed ARC hdr that is
1175 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
1176 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
1177 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR)
1178 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH)
1179 #define HDR_COMPRESSION_ENABLED(hdr) \
1180 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
1182 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE)
1183 #define HDR_L2_READING(hdr) \
1184 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
1185 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
1186 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING)
1187 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
1188 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
1189 #define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
1191 #define HDR_ISTYPE_METADATA(hdr) \
1192 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
1193 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr))
1195 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
1196 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
1201 #define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \
1203 #define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
1241 #define HDR_LOCK(hdr) \
1242 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
1393 static void arc_hdr_free_pdata(arc_buf_hdr_t *hdr);
1401 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1402 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1408 l2arc_trim(const arc_buf_hdr_t *hdr)
1410 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
1412 ASSERT(HDR_HAS_L2HDR(hdr));
1415 if (HDR_GET_PSIZE(hdr) != 0) {
1416 trim_map_free(dev->l2ad_vdev, hdr->b_l2hdr.b_daddr,
1417 HDR_GET_PSIZE(hdr), 0);
1438 #define HDR_EMPTY(hdr) \
1439 ((hdr)->b_dva.dva_word[0] == 0 && \
1440 (hdr)->b_dva.dva_word[1] == 0)
1442 #define HDR_EQUAL(spa, dva, birth, hdr) \
1443 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
1444 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
1445 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1448 buf_discard_identity(arc_buf_hdr_t *hdr)
1450 hdr->b_dva.dva_word[0] = 0;
1451 hdr->b_dva.dva_word[1] = 0;
1452 hdr->b_birth = 0;
1462 arc_buf_hdr_t *hdr;
1465 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
1466 hdr = hdr->b_hash_next) {
1467 if (HDR_EQUAL(spa, dva, birth, hdr)) {
1469 return (hdr);
1485 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1487 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1492 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1493 ASSERT(hdr->b_birth != 0);
1494 ASSERT(!HDR_IN_HASH_TABLE(hdr));
1505 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1509 hdr->b_hash_next = buf_hash_table.ht_table[idx];
1510 buf_hash_table.ht_table[idx] = hdr;
1511 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1529 buf_hash_remove(arc_buf_hdr_t *hdr)
1532 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1535 ASSERT(HDR_IN_HASH_TABLE(hdr));
1538 while ((fhdr = *hdrp) != hdr) {
1542 *hdrp = hdr->b_hash_next;
1543 hdr->b_hash_next = NULL;
1544 arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1583 arc_buf_hdr_t *hdr = vbuf;
1585 bzero(hdr, HDR_FULL_SIZE);
1586 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1587 refcount_create(&hdr->b_l1hdr.b_refcnt);
1588 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1589 multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1599 arc_buf_hdr_t *hdr = vbuf;
1601 bzero(hdr, HDR_L2ONLY_SIZE);
1628 arc_buf_hdr_t *hdr = vbuf;
1630 ASSERT(HDR_EMPTY(hdr));
1631 cv_destroy(&hdr->b_l1hdr.b_cv);
1632 refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1633 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1634 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1642 arc_buf_hdr_t *hdr = vbuf;
1644 ASSERT(HDR_EMPTY(hdr));
1729 arc_cksum_free(arc_buf_hdr_t *hdr)
1731 ASSERT(HDR_HAS_L1HDR(hdr));
1732 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1733 if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1734 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1735 hdr->b_l1hdr.b_freeze_cksum = NULL;
1737 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1743 arc_buf_hdr_t *hdr = buf->b_hdr;
1749 ASSERT(HDR_HAS_L1HDR(hdr));
1751 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1752 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1753 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1756 fletcher_2_native(buf->b_data, HDR_GET_LSIZE(hdr), NULL, &zc);
1757 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1759 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1763 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
1769 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
1783 if (!HDR_COMPRESSION_ENABLED(hdr) && compress != ZIO_COMPRESS_OFF) {
1784 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
1785 uint64_t lsize = HDR_GET_LSIZE(hdr);
1788 void *cbuf = zio_buf_alloc(HDR_GET_PSIZE(hdr));
1790 ASSERT3U(csize, <=, HDR_GET_PSIZE(hdr));
1791 if (csize < HDR_GET_PSIZE(hdr)) {
1798 * we rely on the fact that the hdr's psize
1804 bzero((char *)cbuf + csize, HDR_GET_PSIZE(hdr) - csize);
1805 csize = HDR_GET_PSIZE(hdr);
1807 zio_push_transform(zio, cbuf, csize, HDR_GET_PSIZE(hdr), NULL);
1833 arc_buf_hdr_t *hdr = buf->b_hdr;
1838 ASSERT(HDR_HAS_L1HDR(hdr));
1840 if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1841 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1844 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
1846 hdr), NULL,
1847 hdr->b_l1hdr.b_freeze_cksum);
1848 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1900 arc_buf_type(arc_buf_hdr_t *hdr)
1903 if (HDR_ISTYPE_METADATA(hdr)) {
1908 VERIFY3U(hdr->b_type, ==, type);
1931 arc_buf_hdr_t *hdr = buf->b_hdr;
1934 if (hdr->b_l1hdr.b_state != arc_anon)
1936 if (HDR_IO_IN_PROGRESS(hdr))
1941 ASSERT(HDR_HAS_L1HDR(hdr));
1942 arc_cksum_free(hdr);
1944 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1947 if (hdr->b_l1hdr.b_thawed != NULL)
1948 kmem_free(hdr->b_l1hdr.b_thawed, 1);
1949 hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
1953 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1963 arc_buf_hdr_t *hdr = buf->b_hdr;
1969 hash_lock = HDR_LOCK(hdr);
1972 ASSERT(HDR_HAS_L1HDR(hdr));
1973 ASSERT(hdr->b_l1hdr.b_freeze_cksum != NULL ||
1974 hdr->b_l1hdr.b_state == arc_anon);
1984 * the hash_lock must be held or the hdr must be undiscoverable. This
1989 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1991 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
1992 hdr->b_flags |= flags;
1996 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1998 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
1999 hdr->b_flags &= ~flags;
2010 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
2012 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2021 if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
2022 arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
2023 HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
2024 ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
2025 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
2027 arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
2028 HDR_SET_COMPRESS(hdr, cmp);
2029 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
2030 ASSERT(HDR_COMPRESSION_ENABLED(hdr));
2037 arc_buf_hdr_t *hdr = buf->b_hdr;
2038 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
2042 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
2043 } else if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) {
2049 IMPLY(HDR_COMPRESSION_ENABLED(hdr), HDR_GET_PSIZE(hdr) == 0 ||
2050 HDR_GET_PSIZE(hdr) == HDR_GET_LSIZE(hdr));
2051 ASSERT(!HDR_SHARED_DATA(hdr));
2052 bcopy(hdr->b_l1hdr.b_pdata, buf->b_data, HDR_GET_LSIZE(hdr));
2054 ASSERT(!HDR_SHARED_DATA(hdr));
2055 ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr));
2056 error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
2057 hdr->b_l1hdr.b_pdata, buf->b_data, HDR_GET_PSIZE(hdr),
2058 HDR_GET_LSIZE(hdr));
2060 zfs_dbgmsg("hdr %p, compress %d, psize %d, lsize %d",
2061 hdr, HDR_GET_COMPRESS(hdr), HDR_GET_PSIZE(hdr),
2062 HDR_GET_LSIZE(hdr));
2067 ASSERT(!HDR_SHARED_DATA(hdr));
2069 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
2079 arc_hdr_size(arc_buf_hdr_t *hdr)
2083 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
2084 HDR_GET_PSIZE(hdr) > 0) {
2085 size = HDR_GET_PSIZE(hdr);
2087 ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
2088 size = HDR_GET_LSIZE(hdr);
2095 * We account for the space used by the hdr and the arc buf individually
2099 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
2101 arc_buf_contents_t type = arc_buf_type(hdr);
2102 uint64_t lsize = HDR_GET_LSIZE(hdr);
2104 ASSERT(HDR_HAS_L1HDR(hdr));
2107 ASSERT0(hdr->b_l1hdr.b_bufcnt);
2108 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2109 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2110 (void) refcount_add_many(&state->arcs_esize[type], lsize, hdr);
2115 if (hdr->b_l1hdr.b_pdata != NULL) {
2117 arc_hdr_size(hdr), hdr);
2119 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2131 * We account for the space used by the hdr and the arc buf individually
2135 arc_evitable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
2137 arc_buf_contents_t type = arc_buf_type(hdr);
2138 uint64_t lsize = HDR_GET_LSIZE(hdr);
2140 ASSERT(HDR_HAS_L1HDR(hdr));
2143 ASSERT0(hdr->b_l1hdr.b_bufcnt);
2144 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2145 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2147 lsize, hdr);
2152 if (hdr->b_l1hdr.b_pdata != NULL) {
2154 arc_hdr_size(hdr), hdr);
2156 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2168 * Add a reference to this hdr indicating that someone is actively
2174 add_reference(arc_buf_hdr_t *hdr, void *tag)
2176 ASSERT(HDR_HAS_L1HDR(hdr));
2177 if (!MUTEX_HELD(HDR_LOCK(hdr))) {
2178 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
2179 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2180 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2183 arc_state_t *state = hdr->b_l1hdr.b_state;
2185 if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2189 multilist_remove(&state->arcs_list[arc_buf_type(hdr)],
2190 hdr);
2191 arc_evitable_space_decrement(hdr, state);
2194 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
2199 * Remove a reference from this hdr. When the reference transitions from
2200 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2204 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
2207 arc_state_t *state = hdr->b_l1hdr.b_state;
2209 ASSERT(HDR_HAS_L1HDR(hdr));
2217 if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
2219 multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
2220 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
2221 arc_evictable_space_increment(hdr, state);
2231 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
2238 arc_buf_contents_t buftype = arc_buf_type(hdr);
2241 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
2243 * L1 hdr doesn't always exist when we change state to arc_anon before
2244 * destroying a header, in which case reallocating to add the L1 hdr is
2247 if (HDR_HAS_L1HDR(hdr)) {
2248 old_state = hdr->b_l1hdr.b_state;
2249 refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
2250 bufcnt = hdr->b_l1hdr.b_bufcnt;
2251 update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pdata != NULL);
2271 ASSERT(HDR_HAS_L1HDR(hdr));
2272 multilist_remove(&old_state->arcs_list[buftype], hdr);
2276 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2279 arc_evitable_space_decrement(hdr, old_state);
2289 ASSERT(HDR_HAS_L1HDR(hdr));
2290 multilist_insert(&new_state->arcs_list[buftype], hdr);
2294 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2297 arc_evictable_space_increment(hdr, new_state);
2301 ASSERT(!HDR_EMPTY(hdr));
2302 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
2303 buf_hash_remove(hdr);
2308 ASSERT(HDR_HAS_L1HDR(hdr));
2320 HDR_GET_LSIZE(hdr), hdr);
2321 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2330 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2337 * block with the hdr, the owner of the
2338 * reference belongs to the hdr. Only
2348 HDR_GET_LSIZE(hdr), buf);
2352 if (hdr->b_l1hdr.b_pdata != NULL) {
2354 arc_hdr_size(hdr), hdr);
2362 ASSERT(HDR_HAS_L1HDR(hdr));
2375 HDR_GET_LSIZE(hdr), hdr);
2376 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2385 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2392 * block with the hdr, the owner of the
2393 * reference belongs to the hdr. Only
2403 &old_state->arcs_size, HDR_GET_LSIZE(hdr),
2407 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
2409 &old_state->arcs_size, arc_hdr_size(hdr), hdr);
2413 if (HDR_HAS_L1HDR(hdr))
2414 hdr->b_l1hdr.b_state = new_state;
2488 * Allocate an initial buffer for this hdr, subsequent buffers will
2492 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag)
2496 ASSERT(HDR_HAS_L1HDR(hdr));
2497 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2498 VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2499 hdr->b_type == ARC_BUFC_METADATA);
2501 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2502 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2503 ASSERT0(hdr->b_l1hdr.b_bufcnt);
2506 buf->b_hdr = hdr;
2510 add_reference(hdr, tag);
2513 * We're about to change the hdr's b_flags. We must either
2516 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2519 * If the hdr's data can be shared (no byteswapping, hdr is
2520 * uncompressed, hdr's data is not currently being written to the
2522 * bit in the hdr's b_flags to indicate the hdr is sharing it's
2526 if (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2527 HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF && !HDR_L2_WRITING(hdr)) {
2528 buf->b_data = hdr->b_l1hdr.b_pdata;
2529 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2531 buf->b_data = arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2532 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2533 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2537 hdr->b_l1hdr.b_buf = buf;
2538 hdr->b_l1hdr.b_bufcnt += 1;
2550 arc_buf_hdr_t *hdr = from->b_hdr;
2551 uint64_t size = HDR_GET_LSIZE(hdr);
2553 ASSERT(HDR_HAS_L1HDR(hdr));
2554 ASSERT(hdr->b_l1hdr.b_state != arc_anon);
2557 buf->b_hdr = hdr;
2559 buf->b_next = hdr->b_l1hdr.b_buf;
2560 hdr->b_l1hdr.b_buf = buf;
2561 buf->b_data = arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2563 hdr->b_l1hdr.b_bufcnt += 1;
2565 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2594 arc_buf_hdr_t *hdr = buf->b_hdr;
2597 ASSERT(HDR_HAS_L1HDR(hdr));
2598 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
2599 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2601 atomic_add_64(&arc_loaned_bytes, -HDR_GET_LSIZE(hdr));
2608 arc_buf_hdr_t *hdr = buf->b_hdr;
2611 ASSERT(HDR_HAS_L1HDR(hdr));
2612 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2613 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
2615 atomic_add_64(&arc_loaned_bytes, HDR_GET_LSIZE(hdr));
2632 arc_hdr_free_on_write(arc_buf_hdr_t *hdr)
2634 arc_state_t *state = hdr->b_l1hdr.b_state;
2635 arc_buf_contents_t type = arc_buf_type(hdr);
2636 uint64_t size = arc_hdr_size(hdr);
2639 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2640 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2644 size, hdr);
2646 (void) refcount_remove_many(&state->arcs_size, size, hdr);
2654 l2arc_free_data_on_write(hdr->b_l1hdr.b_pdata, size, type);
2658 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
2659 * data buffer, we transfer the refcount ownership to the hdr and update
2663 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2665 arc_state_t *state = hdr->b_l1hdr.b_state;
2667 ASSERT(!HDR_SHARED_DATA(hdr));
2669 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2670 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2674 * refcount ownership to the hdr since it always owns
2677 refcount_transfer_ownership(&state->arcs_size, buf, hdr);
2678 hdr->b_l1hdr.b_pdata = buf->b_data;
2679 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2682 * Since we've transferred ownership to the hdr we need
2686 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
2687 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
2688 ARCSTAT_INCR(arcstat_overhead_size, -HDR_GET_LSIZE(hdr));
2692 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2694 arc_state_t *state = hdr->b_l1hdr.b_state;
2696 ASSERT(HDR_SHARED_DATA(hdr));
2698 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
2699 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2705 refcount_transfer_ownership(&state->arcs_size, hdr, buf);
2706 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2707 hdr->b_l1hdr.b_pdata = NULL;
2711 * the arc buf and the hdr, count it as overhead.
2713 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
2714 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
2715 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2726 arc_buf_hdr_t *hdr = buf->b_hdr;
2727 uint64_t size = HDR_GET_LSIZE(hdr);
2732 * if we're not sharing this with the hdr. If we are sharing
2733 * it with the hdr, then hdr will have performed the allocation
2738 * We're about to change the hdr's b_flags. We must either
2741 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2750 ASSERT(HDR_SHARED_DATA(hdr));
2751 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2753 arc_free_data_buf(hdr, buf->b_data, size, buf);
2758 ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
2759 hdr->b_l1hdr.b_bufcnt -= 1;
2766 /* remove the buf from the hdr list */
2768 bufp = &hdr->b_l1hdr.b_buf;
2788 * buffer with the hdr, then reassign the hdr's
2791 * on the hdr's buffer list.
2798 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
2799 arc_hdr_free_pdata(hdr);
2803 * last buffer and the hdr. The data would have
2805 * ownership to the hdr since it's now being shared.
2807 arc_share_buf(hdr, lastbuf);
2808 } else if (HDR_SHARED_DATA(hdr)) {
2812 if (hdr->b_l1hdr.b_bufcnt == 0)
2813 arc_cksum_free(hdr);
2821 arc_hdr_alloc_pdata(arc_buf_hdr_t *hdr)
2823 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2824 ASSERT(HDR_HAS_L1HDR(hdr));
2825 ASSERT(!HDR_SHARED_DATA(hdr));
2827 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2828 hdr->b_l1hdr.b_pdata = arc_get_data_buf(hdr, arc_hdr_size(hdr), hdr);
2829 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
2830 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
2832 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
2833 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
2837 arc_hdr_free_pdata(arc_buf_hdr_t *hdr)
2839 ASSERT(HDR_HAS_L1HDR(hdr));
2840 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
2843 * If the hdr is currently being written to the l2arc then
2848 if (HDR_L2_WRITING(hdr)) {
2849 arc_hdr_free_on_write(hdr);
2852 arc_free_data_buf(hdr, hdr->b_l1hdr.b_pdata,
2853 arc_hdr_size(hdr), hdr);
2855 hdr->b_l1hdr.b_pdata = NULL;
2856 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
2858 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
2859 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
2866 arc_buf_hdr_t *hdr;
2871 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
2872 ASSERT(HDR_EMPTY(hdr));
2873 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
2874 ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL);
2875 HDR_SET_PSIZE(hdr, psize);
2876 HDR_SET_LSIZE(hdr, lsize);
2877 hdr->b_spa = spa;
2878 hdr->b_type = type;
2879 hdr->b_flags = 0;
2880 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
2881 arc_hdr_set_compress(hdr, compress);
2883 hdr->b_l1hdr.b_state = arc_anon;
2884 hdr->b_l1hdr.b_arc_access = 0;
2885 hdr->b_l1hdr.b_bufcnt = 0;
2886 hdr->b_l1hdr.b_buf = NULL;
2889 * Allocate the hdr's buffer. This will contain either
2893 arc_hdr_alloc_pdata(hdr);
2894 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2896 return (hdr);
2907 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
2909 ASSERT(HDR_HAS_L2HDR(hdr));
2912 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
2919 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
2920 buf_hash_remove(hdr);
2922 bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
2936 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2937 ASSERT0(hdr->b_l1hdr.b_bufcnt);
2938 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
2947 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
2955 VERIFY(!HDR_L2_WRITING(hdr));
2956 VERIFY3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2959 if (hdr->b_l1hdr.b_thawed != NULL) {
2960 kmem_free(hdr->b_l1hdr.b_thawed, 1);
2961 hdr->b_l1hdr.b_thawed = NULL;
2973 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
2983 list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
2984 list_remove(&dev->l2ad_buflist, hdr);
2996 (void) refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
2999 buf_discard_identity(hdr);
3000 kmem_cache_free(old, hdr);
3012 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
3014 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
3015 arc_buf_t *buf = arc_buf_alloc_impl(hdr, tag);
3021 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3023 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3025 uint64_t asize = arc_hdr_size(hdr);
3028 ASSERT(HDR_HAS_L2HDR(hdr));
3030 list_remove(&dev->l2ad_buflist, hdr);
3033 ARCSTAT_INCR(arcstat_l2_size, -HDR_GET_LSIZE(hdr));
3037 (void) refcount_remove_many(&dev->l2ad_alloc, asize, hdr);
3038 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3042 arc_hdr_destroy(arc_buf_hdr_t *hdr)
3044 if (HDR_HAS_L1HDR(hdr)) {
3045 hdr->b_l1hdr.b_buf == NULL ||
3046 hdr->b_l1hdr.b_bufcnt > 0);
3047 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3048 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3050 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3051 ASSERT(!HDR_IN_HASH_TABLE(hdr));
3053 if (!HDR_EMPTY(hdr))
3054 buf_discard_identity(hdr);
3056 if (HDR_HAS_L2HDR(hdr)) {
3057 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3072 if (HDR_HAS_L2HDR(hdr)) {
3073 l2arc_trim(hdr);
3074 arc_hdr_l2hdr_destroy(hdr);
3081 if (HDR_HAS_L1HDR(hdr)) {
3082 arc_cksum_free(hdr);
3084 while (hdr->b_l1hdr.b_buf != NULL)
3085 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf, B_TRUE);
3088 if (hdr->b_l1hdr.b_thawed != NULL) {
3089 kmem_free(hdr->b_l1hdr.b_thawed, 1);
3090 hdr->b_l1hdr.b_thawed = NULL;
3094 if (hdr->b_l1hdr.b_pdata != NULL) {
3095 arc_hdr_free_pdata(hdr);
3099 ASSERT3P(hdr->b_hash_next, ==, NULL);
3100 if (HDR_HAS_L1HDR(hdr)) {
3101 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3102 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3103 kmem_cache_free(hdr_full_cache, hdr);
3105 kmem_cache_free(hdr_l2only_cache, hdr);
3112 arc_buf_hdr_t *hdr = buf->b_hdr;
3113 kmutex_t *hash_lock = HDR_LOCK(hdr);
3115 if (hdr->b_l1hdr.b_state == arc_anon) {
3116 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
3117 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3118 VERIFY0(remove_reference(hdr, NULL, tag));
3119 arc_hdr_destroy(hdr);
3124 ASSERT3P(hdr, ==, buf->b_hdr);
3125 ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3126 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3127 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3130 (void) remove_reference(hdr, hash_lock, tag);
3154 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
3160 ASSERT(HDR_HAS_L1HDR(hdr));
3162 state = hdr->b_l1hdr.b_state;
3164 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3165 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3174 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
3180 bytes_evicted += HDR_GET_LSIZE(hdr);
3182 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
3184 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
3185 if (HDR_HAS_L2HDR(hdr)) {
3186 ASSERT(hdr->b_l1hdr.b_pdata == NULL);
3191 arc_change_state(arc_l2c_only, hdr, hash_lock);
3196 hdr = arc_hdr_realloc(hdr, hdr_full_cache,
3199 ASSERT(hdr->b_l1hdr.b_pdata == NULL);
3200 arc_change_state(arc_anon, hdr, hash_lock);
3201 arc_hdr_destroy(hdr);
3210 if (HDR_IO_IN_PROGRESS(hdr) ||
3211 ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3212 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
3218 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
3219 while (hdr->b_l1hdr.b_buf) {
3220 arc_buf_t *buf = hdr->b_l1hdr.b_buf;
3226 bytes_evicted += HDR_GET_LSIZE(hdr);
3231 if (HDR_HAS_L2HDR(hdr)) {
3232 ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
3234 if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3236 HDR_GET_LSIZE(hdr));
3239 HDR_GET_LSIZE(hdr));
3243 if (hdr->b_l1hdr.b_bufcnt == 0) {
3244 arc_cksum_free(hdr);
3246 bytes_evicted += arc_hdr_size(hdr);
3249 * If this hdr is being evicted and has a compressed
3254 arc_hdr_free_pdata(hdr);
3256 arc_change_state(evicted_state, hdr, hash_lock);
3257 ASSERT(HDR_IN_HASH_TABLE(hdr));
3258 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
3259 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
3271 arc_buf_hdr_t *hdr;
3280 for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
3281 hdr = multilist_sublist_prev(mls, marker)) {
3288 * forward. Since we're not holding hdr's hash lock, we
3289 * must be very careful and not remove 'hdr' from the
3291 * 'hdr' as not being on a sublist when they call the
3308 if (hdr->b_spa == 0)
3312 if (spa != 0 && hdr->b_spa != spa) {
3317 hash_lock = HDR_LOCK(hdr);
3331 uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
4373 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
4376 arc_state_t *state = hdr->b_l1hdr.b_state;
4377 arc_buf_contents_t type = arc_buf_type(hdr);
4417 VERIFY3U(hdr->b_type, ==, type);
4444 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4445 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4454 if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
4467 arc_free_data_buf(arc_buf_hdr_t *hdr, void *data, uint64_t size, void *tag)
4469 arc_state_t *state = hdr->b_l1hdr.b_state;
4470 arc_buf_contents_t type = arc_buf_type(hdr);
4473 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4474 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4482 VERIFY3U(hdr->b_type, ==, type);
4498 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
4503 ASSERT(HDR_HAS_L1HDR(hdr));
4505 if (hdr->b_l1hdr.b_state == arc_anon) {
4512 ASSERT0(hdr->b_l1hdr.b_arc_access);
4513 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4514 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
4515 arc_change_state(arc_mru, hdr, hash_lock);
4517 } else if (hdr->b_l1hdr.b_state == arc_mru) {
4528 if (HDR_PREFETCH(hdr)) {
4529 if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
4532 &hdr->b_l1hdr.b_arc_node));
4534 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
4537 hdr->b_l1hdr.b_arc_access = now;
4546 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
4552 hdr->b_l1hdr.b_arc_access = now;
4553 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4554 arc_change_state(arc_mfu, hdr, hash_lock);
4557 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
4565 if (HDR_PREFETCH(hdr)) {
4567 if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
4568 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
4569 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
4572 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4575 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4576 arc_change_state(new_state, hdr, hash_lock);
4579 } else if (hdr->b_l1hdr.b_state == arc_mfu) {
4589 if ((HDR_PREFETCH(hdr)) != 0) {
4590 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4592 ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
4595 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4596 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
4604 if (HDR_PREFETCH(hdr)) {
4609 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
4613 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4614 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4615 arc_change_state(new_state, hdr, hash_lock);
4618 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
4623 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4624 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4625 arc_change_state(arc_mfu, hdr, hash_lock);
4656 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
4659 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
4660 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
4662 if (HDR_COMPRESSION_ENABLED(hdr)) {
4663 ASSERT3U(HDR_GET_COMPRESS(hdr), ==,
4666 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
4667 ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
4674 arc_buf_hdr_t *hdr = zio->io_private;
4681 * The hdr was inserted into hash-table and removed from lists
4688 if (HDR_IN_HASH_TABLE(hdr)) {
4689 ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
4690 ASSERT3U(hdr->b_dva.dva_word[0], ==,
4692 ASSERT3U(hdr->b_dva.dva_word[1], ==,
4695 arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,
4698 ASSERT((found == hdr &&
4699 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
4700 (found == hdr && HDR_L2_READING(hdr)));
4708 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
4710 hdr->b_l1hdr.b_byteswap =
4714 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
4718 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
4719 if (l2arc_noprefetch && HDR_PREFETCH(hdr))
4720 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
4722 callback_list = hdr->b_l1hdr.b_acb;
4726 hdr->b_l1hdr.b_state == arc_anon) {
4733 arc_access(hdr, hash_lock);
4749 acb->acb_buf = arc_buf_alloc_impl(hdr,
4757 add_reference(hdr, acb->acb_private);
4762 hdr->b_l1hdr.b_acb = NULL;
4763 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
4769 ASSERT(HDR_PREFETCH(hdr));
4770 ASSERT0(hdr->b_l1hdr.b_bufcnt);
4771 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
4774 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
4778 arc_hdr_verify(hdr, zio->io_bp);
4780 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
4781 if (hdr->b_l1hdr.b_state != arc_anon)
4782 arc_change_state(arc_anon, hdr, hash_lock);
4783 if (HDR_IN_HASH_TABLE(hdr))
4784 buf_hash_remove(hdr);
4785 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
4790 * that the hdr (and hence the cv) might be freed before we get to
4793 cv_broadcast(&hdr->b_l1hdr.b_cv);
4804 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
4805 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
4823 arc_hdr_destroy(hdr);
4849 arc_buf_hdr_t *hdr = NULL;
4863 hdr = buf_hash_find(guid, bp, &hash_lock);
4866 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pdata != NULL) {
4870 if (HDR_IO_IN_PROGRESS(hdr)) {
4872 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
4895 arc_buf_hdr_t *, hdr);
4898 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
4899 arc_hdr_clear_flags(hdr,
4904 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
4922 acb->acb_next = hdr->b_l1hdr.b_acb;
4923 hdr->b_l1hdr.b_acb = acb;
4931 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
4932 hdr->b_l1hdr.b_state == arc_mfu);
4935 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
4943 arc_buf_hdr_t *, hdr);
4946 arc_hdr_clear_flags(hdr,
4956 buf = hdr->b_l1hdr.b_buf;
4958 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
4959 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
4960 buf = arc_buf_alloc_impl(hdr, private);
4963 add_reference(hdr, private);
4969 refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
4970 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
4972 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
4973 arc_access(hdr, hash_lock);
4975 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
4978 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
4979 demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
4993 if (hdr == NULL) {
4997 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
5001 hdr->b_dva = *BP_IDENTITY(bp);
5002 hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
5003 exists = buf_hash_insert(hdr, &hash_lock);
5008 buf_discard_identity(hdr);
5009 arc_hdr_destroy(hdr);
5015 * (and thus didn't have an L1 hdr), we realloc the
5016 * header to add an L1 hdr.
5018 if (!HDR_HAS_L1HDR(hdr)) {
5019 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
5022 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
5023 ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
5024 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5025 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5026 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
5030 * This hdr is in the ghost list so we access it
5038 arc_access(hdr, hash_lock);
5039 arc_hdr_alloc_pdata(hdr);
5041 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
5042 size = arc_hdr_size(hdr);
5045 * If compression is enabled on the hdr, then will do
5046 * RAW I/O and will store the compressed data in the hdr's
5047 * data block. Otherwise, the hdr's data block will contain
5050 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) {
5055 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5057 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5059 arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
5061 arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH);
5062 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
5068 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5069 hdr->b_l1hdr.b_acb = acb;
5070 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5072 if (HDR_HAS_L2HDR(hdr) &&
5073 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
5074 devw = hdr->b_l2hdr.b_dev->l2ad_writing;
5075 addr = hdr->b_l2hdr.b_daddr;
5085 arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
5087 arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
5096 ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
5098 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
5101 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
5102 demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
5128 if (HDR_HAS_L2HDR(hdr) &&
5129 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
5130 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
5134 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
5139 cb->l2rcb_hdr = hdr;
5148 b_data = hdr->b_l1hdr.b_pdata;
5161 ASSERT3U(HDR_GET_COMPRESS(hdr), !=,
5187 arc_buf_hdr_t *, hdr);
5189 if (HDR_L2_WRITING(hdr))
5198 arc_buf_hdr_t *, hdr);
5203 rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pdata, size,
5204 arc_read_done, hdr, priority, zio_flags, zb);
5221 arc_buf_hdr_t *hdr;
5227 hdr = buf_hash_find(guid, bp, &hash_lock);
5228 if (hdr == NULL)
5235 * would still have the ARC_FLAG_IO_IN_PROGRESS flag set on the hdr
5239 * without the dedup flag set. This would have left the hdr in the MRU
5251 * this hdr, then we don't destroy the hdr.
5253 if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) &&
5254 refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) {
5255 arc_change_state(arc_anon, hdr, hash_lock);
5256 arc_hdr_destroy(hdr);
5268 * a new hdr for the buffer.
5273 arc_buf_hdr_t *hdr = buf->b_hdr;
5283 ASSERT(HDR_HAS_L1HDR(hdr));
5290 if (hdr->b_l1hdr.b_state == arc_anon) {
5292 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5293 ASSERT(!HDR_IN_HASH_TABLE(hdr));
5294 ASSERT(!HDR_HAS_L2HDR(hdr));
5295 ASSERT(HDR_EMPTY(hdr));
5296 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
5297 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
5298 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
5300 hdr->b_l1hdr.b_arc_access = 0;
5304 * have a hdr that is not empty.
5306 buf_discard_identity(hdr);
5312 kmutex_t *hash_lock = HDR_LOCK(hdr);
5320 arc_state_t *state = hdr->b_l1hdr.b_state;
5321 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
5325 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) > 0);
5327 if (HDR_HAS_L2HDR(hdr)) {
5328 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
5338 if (HDR_HAS_L2HDR(hdr)) {
5339 l2arc_trim(hdr);
5340 arc_hdr_l2hdr_destroy(hdr);
5343 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
5349 if (hdr->b_l1hdr.b_bufcnt > 1) {
5352 uint64_t spa = hdr->b_spa;
5353 uint64_t psize = HDR_GET_PSIZE(hdr);
5354 uint64_t lsize = HDR_GET_LSIZE(hdr);
5355 enum zio_compress compress = HDR_GET_COMPRESS(hdr);
5356 arc_buf_contents_t type = arc_buf_type(hdr);
5357 VERIFY3U(hdr->b_type, ==, type);
5359 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
5360 (void) remove_reference(hdr, hash_lock, tag);
5363 ASSERT(HDR_SHARED_DATA(hdr));
5364 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
5369 * Pull the data off of this hdr and attach it to
5370 * a new anonymous hdr. Also find the last buffer
5371 * in the hdr's buffer list.
5374 bufp = &hdr->b_l1hdr.b_buf;
5395 * If the current arc_buf_t and the hdr are sharing their data
5398 * of the hdr's b_buf list.
5401 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
5411 arc_unshare_buf(hdr, buf);
5412 arc_share_buf(hdr, lastbuf);
5414 } else if (HDR_SHARED_DATA(hdr)) {
5417 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
5421 HDR_GET_LSIZE(hdr), buf);
5423 if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
5426 HDR_GET_LSIZE(hdr), buf);
5429 hdr->b_l1hdr.b_bufcnt -= 1;
5438 * Allocate a new hdr. The new hdr will contain a b_pdata
5458 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
5459 /* protected by hash lock, or hdr is on arc_anon */
5460 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
5461 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5462 arc_change_state(arc_anon, hdr, hash_lock);
5463 hdr->b_l1hdr.b_arc_access = 0;
5466 buf_discard_identity(hdr);
5501 arc_buf_hdr_t *hdr = buf->b_hdr;
5504 ASSERT(HDR_HAS_L1HDR(hdr));
5506 ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
5514 arc_cksum_free(hdr);
5518 if (hdr->b_l1hdr.b_pdata != NULL) {
5520 ASSERT(HDR_SHARED_DATA(hdr));
5522 arc_unshare_buf(hdr, buf);
5524 arc_hdr_free_pdata(hdr);
5528 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
5529 ASSERT(!HDR_SHARED_DATA(hdr));
5534 if (HDR_IO_IN_PROGRESS(hdr))
5538 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5544 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(zio->io_bp));
5547 HDR_SET_PSIZE(hdr, psize);
5548 arc_hdr_set_compress(hdr, compress);
5551 * If the hdr is compressed, then copy the compressed
5553 * data buf into the hdr. Ideally, we would like to always copy the
5556 * in the hdr's b_pdata field.
5558 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) {
5561 arc_hdr_alloc_pdata(hdr);
5562 bcopy(zio->io_data, hdr->b_l1hdr.b_pdata, psize);
5565 ASSERT3U(zio->io_orig_size, ==, HDR_GET_LSIZE(hdr));
5566 ASSERT3U(hdr->b_l1hdr.b_byteswap, ==, DMU_BSWAP_NUMFUNCS);
5567 ASSERT(!HDR_SHARED_DATA(hdr));
5569 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
5570 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
5573 * This hdr is not compressed so we're able to share
5574 * the arc_buf_t data buffer with the hdr.
5576 arc_share_buf(hdr, buf);
5577 VERIFY0(bcmp(zio->io_orig_data, hdr->b_l1hdr.b_pdata,
5578 HDR_GET_LSIZE(hdr)));
5580 arc_hdr_verify(hdr, zio->io_bp);
5609 arc_buf_hdr_t *hdr = buf->b_hdr;
5611 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5614 arc_hdr_verify(hdr, zio->io_bp);
5617 buf_discard_identity(hdr);
5619 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
5620 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
5623 ASSERT(HDR_EMPTY(hdr));
5632 if (!HDR_EMPTY(hdr)) {
5640 exists = buf_hash_insert(hdr, &hash_lock);
5649 panic("bad overwrite, hdr=%p exists=%p",
5650 (void *)hdr, (void *)exists);
5656 exists = buf_hash_insert(hdr, &hash_lock);
5662 panic("bad nopwrite, hdr=%p exists=%p",
5663 (void *)hdr, (void *)exists);
5666 ASSERT(hdr->b_l1hdr.b_bufcnt == 1);
5667 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
5672 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5674 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
5675 arc_access(hdr, hash_lock);
5678 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5681 ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5694 arc_buf_hdr_t *hdr = buf->b_hdr;
5700 ASSERT(!HDR_IO_ERROR(hdr));
5701 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5702 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5703 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
5705 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5715 * The hdr's b_pdata is now stale, free it now. A new data block
5718 if (hdr->b_l1hdr.b_pdata != NULL) {
5721 * the hdr then we need to break that relationship here.
5722 * The hdr will remain with a NULL data pointer and the
5727 arc_unshare_buf(hdr, buf);
5729 arc_hdr_free_pdata(hdr);
5732 arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);
5735 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
5737 zio = zio_write(pio, spa, txg, bp, buf->b_data, HDR_GET_LSIZE(hdr), zp,
5902 arc_buf_hdr_t *hdr = obj;
5909 ASSERT(!HDR_EMPTY(hdr));
5923 return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
6468 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
6477 if (hdr->b_spa != spa_guid) {
6481 if (HDR_HAS_L2HDR(hdr)) {
6485 if (HDR_IO_IN_PROGRESS(hdr)) {
6489 if (!HDR_L2CACHE(hdr)) {
6642 arc_buf_hdr_t *head, *hdr, *hdr_prev;
6665 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
6666 hdr_prev = list_prev(buflist, hdr);
6668 hash_lock = HDR_LOCK(hdr);
6689 list_insert_after(buflist, hdr, head);
6709 ASSERT(HDR_HAS_L1HDR(hdr));
6715 list_remove(buflist, hdr);
6716 l2arc_trim(hdr);
6717 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
6719 ARCSTAT_INCR(arcstat_l2_asize, -arc_hdr_size(hdr));
6720 ARCSTAT_INCR(arcstat_l2_size, -HDR_GET_LSIZE(hdr));
6722 bytes_dropped += arc_hdr_size(hdr);
6724 arc_hdr_size(hdr), hdr);
6731 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING);
6757 arc_buf_hdr_t *hdr;
6768 hdr = cb->l2rcb_hdr;
6769 ASSERT3P(hdr, !=, NULL);
6771 hash_lock = HDR_LOCK(hdr);
6773 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
6780 ASSERT3U(arc_hdr_size(hdr), <, zio->io_size);
6782 bcopy(cb->l2rcb_data, hdr->b_l1hdr.b_pdata,
6783 arc_hdr_size(hdr));
6798 zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);
6799 zio->io_data = zio->io_orig_data = hdr->b_l1hdr.b_pdata;
6807 ASSERT3P(zio->io_data, ==, hdr->b_l1hdr.b_pdata);
6811 valid_cksum = arc_cksum_is_equal(hdr, zio);
6812 if (valid_cksum && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
6814 zio->io_private = hdr;
6841 hdr->b_l1hdr.b_pdata, zio->io_size, arc_read_done,
6842 hdr, zio->io_priority, cb->l2rcb_flags,
6903 hdr, *hdr_prev;
6931 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
6932 hdr_prev = list_prev(buflist, hdr);
6934 hash_lock = HDR_LOCK(hdr);
6952 if (HDR_L2_WRITE_HEAD(hdr)) {
6957 list_remove(buflist, hdr);
6962 if (!all && HDR_HAS_L2HDR(hdr) &&
6963 (hdr->b_l2hdr.b_daddr >= taddr ||
6964 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
6973 ASSERT(HDR_HAS_L2HDR(hdr));
6974 if (!HDR_HAS_L1HDR(hdr)) {
6975 ASSERT(!HDR_L2_READING(hdr));
6981 arc_change_state(arc_anon, hdr, hash_lock);
6982 arc_hdr_destroy(hdr);
6984 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
6991 if (HDR_L2_READING(hdr)) {
6993 arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED);
6997 ASSERT(!HDR_L2_WRITING(hdr));
6999 arc_hdr_l2hdr_destroy(hdr);
7020 arc_buf_hdr_t *hdr, *hdr_prev, *head;
7053 hdr = multilist_sublist_head(mls);
7055 hdr = multilist_sublist_tail(mls);
7056 if (hdr == NULL)
7063 for (; hdr; hdr = hdr_prev) {
7067 hdr_prev = multilist_sublist_next(mls, hdr);
7069 hdr_prev = multilist_sublist_prev(mls, hdr);
7071 HDR_GET_LSIZE(hdr));
7073 hash_lock = HDR_LOCK(hdr);
7082 passed_sz += HDR_GET_LSIZE(hdr);
7092 if (!l2arc_write_eligible(guid, hdr)) {
7103 ASSERT(HDR_HAS_L1HDR(hdr));
7105 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);
7106 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
7107 ASSERT3U(arc_hdr_size(hdr), >, 0);
7108 uint64_t size = arc_hdr_size(hdr);
7138 hdr->b_l2hdr.b_dev = dev;
7139 hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
7140 arc_hdr_set_flags(hdr,
7144 list_insert_head(&dev->l2ad_buflist, hdr);
7147 (void) refcount_add_many(&dev->l2ad_alloc, size, hdr);
7150 * Normally the L2ARC can use the hdr's data, but if
7151 * we're sharing data between the hdr and one of its
7159 if (!HDR_SHARED_DATA(hdr) && size == asize) {
7160 to_write = hdr->b_l1hdr.b_pdata;
7162 arc_buf_contents_t type = arc_buf_type(hdr);
7170 bcopy(hdr->b_l1hdr.b_pdata, to_write, size);
7176 hdr->b_l2hdr.b_daddr, asize, to_write,
7177 ZIO_CHECKSUM_OFF, NULL, hdr,
7181 write_sz += HDR_GET_LSIZE(hdr);
Indexes created Fri May 15 00:25:31 UTC 2026