Home | History | Annotate | Line # | Download | only in serve-stale
tests.sh revision 1.1.1.15
      1 #!/bin/sh
      2 
      3 # Copyright (C) Internet Systems Consortium, Inc. ("ISC")
      4 #
      5 # SPDX-License-Identifier: MPL-2.0
      6 #
      7 # This Source Code Form is subject to the terms of the Mozilla Public
      8 # License, v. 2.0.  If a copy of the MPL was not distributed with this
      9 # file, you can obtain one at https://mozilla.org/MPL/2.0/.
     10 #
     11 # See the COPYRIGHT file distributed with this work for additional
     12 # information regarding copyright ownership.
     13 
     14 set -e
     15 
     16 . ../conf.sh
     17 
     18 RNDCCMD="$RNDC -c ../_common/rndc.conf -p ${CONTROLPORT} -s"
     19 DIG="$DIG +time=12 +tries=1"
     20 
     21 max_stale_ttl=$(sed -ne 's,^[[:space:]]*max-stale-ttl \([[:digit:]]*\).*,\1,p' $TOP_SRCDIR/bin/named/config.c)
     22 stale_answer_ttl=$(sed -ne 's,^[[:space:]]*stale-answer-ttl \([[:digit:]]*\).*,\1,p' $TOP_SRCDIR/bin/named/config.c)
     23 
     24 status=0
     25 n=0
     26 
     27 #
     28 # First test server with serve-stale options set.
     29 #
     30 echo_i "test server with serve-stale options set"
     31 
     32 n=$((n + 1))
     33 echo_i "prime cache longttl.example TXT ($n)"
     34 ret=0
     35 $DIG -p ${PORT} @10.53.0.1 longttl.example TXT >dig.out.test$n || ret=1
     36 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
     37 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
     38 if [ $ret != 0 ]; then echo_i "failed"; fi
     39 status=$((status + ret))
     40 
     41 n=$((n + 1))
     42 echo_i "prime cache data.example TXT ($n)"
     43 ret=0
     44 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$n || ret=1
     45 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
     46 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
     47 if [ $ret != 0 ]; then echo_i "failed"; fi
     48 status=$((status + ret))
     49 
     50 n=$((n + 1))
     51 echo_i "prime cache othertype.example CAA ($n)"
     52 ret=0
     53 $DIG -p ${PORT} @10.53.0.1 othertype.example CAA >dig.out.test$n || ret=1
     54 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
     55 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
     56 if [ $ret != 0 ]; then echo_i "failed"; fi
     57 status=$((status + ret))
     58 
     59 n=$((n + 1))
     60 echo_i "prime cache nodata.example TXT ($n)"
     61 ret=0
     62 $DIG -p ${PORT} @10.53.0.1 nodata.example TXT >dig.out.test$n || ret=1
     63 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
     64 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
     65 if [ $ret != 0 ]; then echo_i "failed"; fi
     66 status=$((status + ret))
     67 
     68 n=$((n + 1))
     69 echo_i "prime cache nxdomain.example TXT ($n)"
     70 ret=0
     71 $DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT >dig.out.test$n || ret=1
     72 grep "status: NXDOMAIN" dig.out.test$n >/dev/null || ret=1
     73 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
     74 if [ $ret != 0 ]; then echo_i "failed"; fi
     75 status=$((status + ret))
     76 
     77 n=$((n + 1))
     78 echo_i "verify prime cache statistics ($n)"
     79 ret=0
     80 rm -f ns1/named.stats
     81 $RNDCCMD 10.53.0.1 stats >/dev/null 2>&1
     82 [ -f ns1/named.stats ] || ret=1
     83 cp ns1/named.stats ns1/named.stats.$n
     84 # Check first 10 lines of Cache DB statistics.  After prime queries, we expect
     85 # two active TXT, one active Others, one nxrrset TXT, and one NXDOMAIN.
     86 grep -A 10 "++ Cache DB RRsets ++" ns1/named.stats.$n >ns1/named.stats.$n.cachedb || ret=1
     87 grep "1 Others" ns1/named.stats.$n.cachedb >/dev/null || ret=1
     88 grep "2 TXT" ns1/named.stats.$n.cachedb >/dev/null || ret=1
     89 grep "1 !TXT" ns1/named.stats.$n.cachedb >/dev/null || ret=1
     90 grep "1 NXDOMAIN" ns1/named.stats.$n.cachedb >/dev/null || ret=1
     91 if [ $ret != 0 ]; then echo_i "failed"; fi
     92 status=$((status + ret))
     93 
     94 n=$((n + 1))
     95 echo_i "disable responses from authoritative server ($n)"
     96 ret=0
     97 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
     98 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
     99 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
    100 if [ $ret != 0 ]; then echo_i "failed"; fi
    101 status=$((status + ret))
    102 
    103 n=$((n + 1))
    104 echo_i "check 'rndc serve-stale status' ($n)"
    105 ret=0
    106 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    107 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=4 max-stale-ttl=3600 stale-refresh-time=30)' rndc.out.test$n >/dev/null || ret=1
    108 if [ $ret != 0 ]; then echo_i "failed"; fi
    109 status=$((status + ret))
    110 
    111 sleep 2
    112 
    113 # Run rndc dumpdb, test whether the stale data has correct comment printed.
    114 # The max-stale-ttl is 3600 seconds, so the comment should say the data is
    115 # stale for somewhere between 3500-3599 seconds.
    116 echo_i "check rndc dump stale data.example ($n)"
    117 rndc_dumpdb ns1 || ret=1
    118 awk '/; stale/ { x=$0; getline; print x, $0}' ns1/named_dump.db.test$n \
    119   | grep "; stale data\.example.*3[56]...*TXT.*A text record with a 2 second ttl" >/dev/null 2>&1 || ret=1
    120 # Also make sure the not expired data does not have a stale comment.
    121 awk '/; authanswer/ { x=$0; getline; print x, $0}' ns1/named_dump.db.test$n \
    122   | grep "; authanswer longttl\.example.*[56]...*TXT.*A text record with a 600 second ttl" >/dev/null 2>&1 || ret=1
    123 if [ $ret != 0 ]; then echo_i "failed"; fi
    124 status=$((status + ret))
    125 
    126 echo_i "sending queries for tests $((n + 1))-$((n + 5))..."
    127 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$((n + 1)) &
    128 $DIG -p ${PORT} @10.53.0.1 longttl.example TXT >dig.out.test$((n + 2)) &
    129 $DIG -p ${PORT} @10.53.0.1 othertype.example CAA >dig.out.test$((n + 3)) &
    130 $DIG -p ${PORT} @10.53.0.1 nodata.example TXT >dig.out.test$((n + 4)) &
    131 $DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT >dig.out.test$((n + 5)) &
    132 
    133 wait
    134 
    135 n=$((n + 1))
    136 echo_i "check stale data.example TXT ($n)"
    137 ret=0
    138 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    139 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    140 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    141 grep "data\.example\..*4.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    142 if [ $ret != 0 ]; then echo_i "failed"; fi
    143 status=$((status + ret))
    144 
    145 n=$((n + 1))
    146 echo_i "check non-stale longttl.example TXT ($n)"
    147 ret=0
    148 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    149 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    150 grep "EDE" dig.out.test$n >/dev/null && ret=1
    151 grep "longttl\.example\..*59[0-9].*IN.*TXT.*A text record with a 600 second ttl" dig.out.test$n >/dev/null || ret=1
    152 if [ $ret != 0 ]; then echo_i "failed"; fi
    153 status=$((status + ret))
    154 
    155 n=$((n + 1))
    156 echo_i "check stale othertype.example CAA ($n)"
    157 ret=0
    158 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    159 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    160 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    161 grep "othertype\.example\..*4.*IN.*CAA.*0.*issue" dig.out.test$n >/dev/null || ret=1
    162 if [ $ret != 0 ]; then echo_i "failed"; fi
    163 status=$((status + ret))
    164 
    165 n=$((n + 1))
    166 echo_i "check stale nodata.example TXT ($n)"
    167 ret=0
    168 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    169 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    170 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    171 grep "example\..*4.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
    172 if [ $ret != 0 ]; then echo_i "failed"; fi
    173 status=$((status + ret))
    174 
    175 n=$((n + 1))
    176 echo_i "check stale nxdomain.example TXT ($n)"
    177 ret=0
    178 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    179 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    180 if [ $ret != 0 ]; then echo_i "failed"; fi
    181 status=$((status + ret))
    182 
    183 n=$((n + 1))
    184 echo_i "verify stale cache statistics ($n)"
    185 ret=0
    186 rm -f ns1/named.stats
    187 $RNDCCMD 10.53.0.1 stats >/dev/null 2>&1
    188 [ -f ns1/named.stats ] || ret=1
    189 cp ns1/named.stats ns1/named.stats.$n
    190 # Check first 10 lines of Cache DB statistics.  After serve-stale queries, we
    191 # expect one active TXT RRset, one stale TXT, one stale nxrrset TXT, and one
    192 # stale NXDOMAIN.
    193 grep -A 10 "++ Cache DB RRsets ++" ns1/named.stats.$n >ns1/named.stats.$n.cachedb || ret=1
    194 grep "1 TXT" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    195 grep "1 #Others" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    196 grep "1 #TXT" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    197 grep "1 #!TXT" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    198 status=$((status + ret))
    199 if [ $ret != 0 ]; then echo_i "failed"; fi
    200 
    201 # Test stale-refresh-time when serve-stale is enabled via configuration.
    202 # Steps for testing stale-refresh-time option (default).
    203 # 1. Prime cache data.example txt
    204 # 2. Disable responses from authoritative server.
    205 # 3. Sleep for TTL duration so rrset TTL expires (2 sec)
    206 # 4. Query data.example
    207 # 5. Check if response come from stale rrset (4 sec TTL)
    208 # 6. Enable responses from authoritative server.
    209 # 7. Query data.example
    210 # 8. Check if response come from stale rrset, since the query
    211 #    is still within stale-refresh-time window.
    212 n=$((n + 1))
    213 echo_i "check 'rndc serve-stale status' ($n)"
    214 ret=0
    215 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    216 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=4 max-stale-ttl=3600 stale-refresh-time=30)' rndc.out.test$n >/dev/null || ret=1
    217 if [ $ret != 0 ]; then echo_i "failed"; fi
    218 status=$((status + ret))
    219 
    220 # Step 1-3 done above.
    221 
    222 # Step 4.
    223 n=$((n + 1))
    224 echo_i "sending query for test ($n)"
    225 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$n || ret=1
    226 
    227 # Step 5.
    228 echo_i "check stale data.example TXT (stale-refresh-time) ($n)"
    229 ret=0
    230 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    231 grep "EDE: 3 (Stale Answer): (query within stale refresh time window)" dig.out.test$n >/dev/null || ret=1
    232 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    233 grep "data\.example\..*4.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    234 if [ $ret != 0 ]; then echo_i "failed"; fi
    235 status=$((status + ret))
    236 
    237 # Step 6.
    238 n=$((n + 1))
    239 echo_i "enable responses from authoritative server ($n)"
    240 ret=0
    241 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
    242 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    243 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
    244 if [ $ret != 0 ]; then echo_i "failed"; fi
    245 status=$((status + ret))
    246 
    247 # Step 7.
    248 echo_i "sending query for test $((n + 1))"
    249 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$((n + 1)) || true
    250 
    251 # Step 8.
    252 n=$((n + 1))
    253 echo_i "check stale data.example TXT comes from cache (stale-refresh-time) ($n)"
    254 ret=0
    255 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    256 grep "EDE: 3 (Stale Answer): (query within stale refresh time window)" dig.out.test$n >/dev/null || ret=1
    257 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    258 grep "data\.example\..*4.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    259 if [ $ret != 0 ]; then echo_i "failed"; fi
    260 status=$((status + ret))
    261 
    262 #
    263 # Test interaction with local zone
    264 #
    265 
    266 n=$((n + 1))
    267 echo_i "check that serve-stale does not recurse for local authoritative zone ($n)"
    268 ret=0
    269 
    270 num=0
    271 threshold=10
    272 while [ $num -lt $threshold ]; do
    273 
    274   echo_i "dig test.serve.stale TXT ($n)"
    275   $DIG -p ${PORT} @10.53.0.3 test.serve.stale TXT >dig.out.test$n.$num || ret=1
    276   grep "status: SERVFAIL" dig.out.test$n.$num >/dev/null || ret=1
    277   if [ $ret != 0 ]; then num=$threshold; fi
    278 
    279   sleep 1
    280   num=$((num + 1))
    281 done
    282 if [ $ret != 0 ]; then echo_i "failed"; fi
    283 status=$((status + ret))
    284 
    285 #
    286 # Test disabling serve-stale via rndc.
    287 #
    288 
    289 n=$((n + 1))
    290 echo_i "updating ns1/named.conf ($n)"
    291 ret=0
    292 copy_setports ns1/named2.conf.in ns1/named.conf
    293 if [ $ret != 0 ]; then echo_i "failed"; fi
    294 status=$((status + ret))
    295 
    296 n=$((n + 1))
    297 echo_i "running 'rndc reload' ($n)"
    298 ret=0
    299 rndc_reload ns1 10.53.0.1
    300 if [ $ret != 0 ]; then echo_i "failed"; fi
    301 status=$((status + ret))
    302 
    303 n=$((n + 1))
    304 echo_i "check 'rndc serve-stale status' ($n)"
    305 ret=0
    306 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    307 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=4 max-stale-ttl=3600 stale-refresh-time=0)' rndc.out.test$n >/dev/null || ret=1
    308 if [ $ret != 0 ]; then echo_i "failed"; fi
    309 status=$((status + ret))
    310 
    311 n=$((n + 1))
    312 echo_i "disable responses from authoritative server ($n)"
    313 ret=0
    314 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
    315 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    316 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
    317 if [ $ret != 0 ]; then echo_i "failed"; fi
    318 status=$((status + ret))
    319 
    320 n=$((n + 1))
    321 echo_i "running 'rndc serve-stale off' ($n)"
    322 ret=0
    323 $RNDCCMD 10.53.0.1 serve-stale off || ret=1
    324 if [ $ret != 0 ]; then echo_i "failed"; fi
    325 status=$((status + ret))
    326 
    327 n=$((n + 1))
    328 echo_i "check 'rndc serve-stale status' ($n)"
    329 ret=0
    330 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    331 grep '_default: stale cache enabled; stale answers disabled (stale-answer-ttl=4 max-stale-ttl=3600 stale-refresh-time=0)' rndc.out.test$n >/dev/null || ret=1
    332 if [ $ret != 0 ]; then echo_i "failed"; fi
    333 status=$((status + ret))
    334 
    335 echo_i "sending queries for tests $((n + 1))-$((n + 4))..."
    336 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$((n + 1)) &
    337 $DIG -p ${PORT} @10.53.0.1 othertype.example CAA >dig.out.test$((n + 2)) &
    338 $DIG -p ${PORT} @10.53.0.1 nodata.example TXT >dig.out.test$((n + 3)) &
    339 $DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT >dig.out.test$((n + 4)) &
    340 
    341 wait
    342 
    343 n=$((n + 1))
    344 echo_i "check stale data.example TXT (serve-stale off) ($n)"
    345 ret=0
    346 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    347 grep "EDE" dig.out.test$n >/dev/null && ret=1
    348 if [ $ret != 0 ]; then echo_i "failed"; fi
    349 status=$((status + ret))
    350 
    351 n=$((n + 1))
    352 echo_i "check stale othertype.example CAA (serve-stale off) ($n)"
    353 ret=0
    354 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    355 grep "EDE" dig.out.test$n >/dev/null && ret=1
    356 if [ $ret != 0 ]; then echo_i "failed"; fi
    357 status=$((status + ret))
    358 
    359 n=$((n + 1))
    360 echo_i "check stale nodata.example TXT (serve-stale off) ($n)"
    361 ret=0
    362 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    363 grep "EDE" dig.out.test$n >/dev/null && ret=1
    364 if [ $ret != 0 ]; then echo_i "failed"; fi
    365 status=$((status + ret))
    366 
    367 n=$((n + 1))
    368 echo_i "check stale nxdomain.example TXT (serve-stale off) ($n)"
    369 ret=0
    370 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    371 grep "EDE" dig.out.test$n >/dev/null && ret=1
    372 if [ $ret != 0 ]; then echo_i "failed"; fi
    373 status=$((status + ret))
    374 
    375 #
    376 # Test enabling serve-stale via rndc.
    377 #
    378 n=$((n + 1))
    379 echo_i "running 'rndc serve-stale on' ($n)"
    380 ret=0
    381 $RNDCCMD 10.53.0.1 serve-stale on || ret=1
    382 if [ $ret != 0 ]; then echo_i "failed"; fi
    383 status=$((status + ret))
    384 
    385 n=$((n + 1))
    386 echo_i "check 'rndc serve-stale status' ($n)"
    387 ret=0
    388 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    389 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=4 max-stale-ttl=3600 stale-refresh-time=0)' rndc.out.test$n >/dev/null || ret=1
    390 if [ $ret != 0 ]; then echo_i "failed"; fi
    391 status=$((status + ret))
    392 
    393 echo_i "sending queries for tests $((n + 1))-$((n + 4))..."
    394 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$((n + 1)) &
    395 $DIG -p ${PORT} @10.53.0.1 othertype.example CAA >dig.out.test$((n + 2)) &
    396 $DIG -p ${PORT} @10.53.0.1 nodata.example TXT >dig.out.test$((n + 3)) &
    397 $DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT >dig.out.test$((n + 4)) &
    398 
    399 wait
    400 
    401 n=$((n + 1))
    402 echo_i "check stale data.example TXT (serve-stale on) ($n)"
    403 ret=0
    404 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    405 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    406 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    407 grep "data\.example\..*4.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    408 if [ $ret != 0 ]; then echo_i "failed"; fi
    409 status=$((status + ret))
    410 
    411 n=$((n + 1))
    412 echo_i "check stale othertype.example CAA (serve-stale on) ($n)"
    413 ret=0
    414 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    415 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    416 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    417 grep "othertype\.example\..*4.*IN.*CAA.*0.*issue" dig.out.test$n >/dev/null || ret=1
    418 if [ $ret != 0 ]; then echo_i "failed"; fi
    419 status=$((status + ret))
    420 
    421 n=$((n + 1))
    422 echo_i "check stale nodata.example TXT (serve-stale on) ($n)"
    423 ret=0
    424 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    425 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    426 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    427 grep "example\..*4.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
    428 if [ $ret != 0 ]; then echo_i "failed"; fi
    429 status=$((status + ret))
    430 
    431 n=$((n + 1))
    432 echo_i "check stale nxdomain.example TXT (serve-stale on) ($n)"
    433 ret=0
    434 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    435 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    436 if [ $ret != 0 ]; then echo_i "failed"; fi
    437 status=$((status + ret))
    438 
    439 n=$((n + 1))
    440 echo_i "running 'rndc serve-stale off' ($n)"
    441 ret=0
    442 $RNDCCMD 10.53.0.1 serve-stale off || ret=1
    443 if [ $ret != 0 ]; then echo_i "failed"; fi
    444 status=$((status + ret))
    445 
    446 n=$((n + 1))
    447 echo_i "running 'rndc serve-stale reset' ($n)"
    448 ret=0
    449 $RNDCCMD 10.53.0.1 serve-stale reset || ret=1
    450 if [ $ret != 0 ]; then echo_i "failed"; fi
    451 status=$((status + ret))
    452 
    453 n=$((n + 1))
    454 echo_i "check 'rndc serve-stale status' ($n)"
    455 ret=0
    456 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    457 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=4 max-stale-ttl=3600 stale-refresh-time=0)' rndc.out.test$n >/dev/null || ret=1
    458 if [ $ret != 0 ]; then echo_i "failed"; fi
    459 status=$((status + ret))
    460 
    461 echo_i "sending queries for tests $((n + 1))-$((n + 4))..."
    462 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$((n + 1)) &
    463 $DIG -p ${PORT} @10.53.0.1 othertype.example CAA >dig.out.test$((n + 2)) &
    464 $DIG -p ${PORT} @10.53.0.1 nodata.example TXT >dig.out.test$((n + 3)) &
    465 $DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT >dig.out.test$((n + 4)) &
    466 
    467 wait
    468 
    469 n=$((n + 1))
    470 echo_i "check stale data.example TXT (serve-stale reset) ($n)"
    471 ret=0
    472 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    473 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    474 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    475 grep "data\.example\..*4.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    476 if [ $ret != 0 ]; then echo_i "failed"; fi
    477 status=$((status + ret))
    478 
    479 n=$((n + 1))
    480 echo_i "check stale othertype.example CAA (serve-stale reset) ($n)"
    481 ret=0
    482 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    483 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    484 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    485 grep "othertype.example\..*4.*IN.*CAA.*0.*issue" dig.out.test$n >/dev/null || ret=1
    486 if [ $ret != 0 ]; then echo_i "failed"; fi
    487 status=$((status + ret))
    488 
    489 n=$((n + 1))
    490 echo_i "check stale nodata.example TXT (serve-stale reset) ($n)"
    491 ret=0
    492 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    493 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    494 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    495 grep "example\..*4.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
    496 if [ $ret != 0 ]; then echo_i "failed"; fi
    497 status=$((status + ret))
    498 
    499 n=$((n + 1))
    500 echo_i "check stale nxdomain.example TXT (serve-stale reset) ($n)"
    501 ret=0
    502 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    503 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    504 if [ $ret != 0 ]; then echo_i "failed"; fi
    505 status=$((status + ret))
    506 
    507 n=$((n + 1))
    508 echo_i "running 'rndc serve-stale off' ($n)"
    509 ret=0
    510 $RNDCCMD 10.53.0.1 serve-stale off || ret=1
    511 if [ $ret != 0 ]; then echo_i "failed"; fi
    512 status=$((status + ret))
    513 
    514 n=$((n + 1))
    515 echo_i "check 'rndc serve-stale status' ($n)"
    516 ret=0
    517 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    518 grep '_default: stale cache enabled; stale answers disabled (stale-answer-ttl=4 max-stale-ttl=3600 stale-refresh-time=0)' rndc.out.test$n >/dev/null || ret=1
    519 if [ $ret != 0 ]; then echo_i "failed"; fi
    520 status=$((status + ret))
    521 
    522 #
    523 # Update named.conf.
    524 # Test server with low max-stale-ttl.
    525 #
    526 echo_i "test server with serve-stale options set, low max-stale-ttl"
    527 
    528 n=$((n + 1))
    529 echo_i "updating ns1/named.conf ($n)"
    530 ret=0
    531 copy_setports ns1/named3.conf.in ns1/named.conf
    532 if [ $ret != 0 ]; then echo_i "failed"; fi
    533 status=$((status + ret))
    534 
    535 n=$((n + 1))
    536 echo_i "running 'rndc reload' ($n)"
    537 ret=0
    538 rndc_reload ns1 10.53.0.1
    539 if [ $ret != 0 ]; then echo_i "failed"; fi
    540 status=$((status + ret))
    541 
    542 n=$((n + 1))
    543 echo_i "check 'rndc serve-stale status' ($n)"
    544 ret=0
    545 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    546 grep '_default: stale cache enabled; stale answers disabled (stale-answer-ttl=3 max-stale-ttl=20 stale-refresh-time=30)' rndc.out.test$n >/dev/null || ret=1
    547 if [ $ret != 0 ]; then echo_i "failed"; fi
    548 status=$((status + ret))
    549 
    550 n=$((n + 1))
    551 echo_i "flush cache, re-enable serve-stale and query again ($n)"
    552 ret=0
    553 $RNDCCMD 10.53.0.1 flushtree example >rndc.out.test$n.1 2>&1 || ret=1
    554 $RNDCCMD 10.53.0.1 serve-stale on >rndc.out.test$n.2 2>&1 || ret=1
    555 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$n || ret=1
    556 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    557 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    558 if [ $ret != 0 ]; then echo_i "failed"; fi
    559 status=$((status + ret))
    560 
    561 n=$((n + 1))
    562 echo_i "check 'rndc serve-stale status' ($n)"
    563 ret=0
    564 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    565 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=3 max-stale-ttl=20 stale-refresh-time=30)' rndc.out.test$n >/dev/null || ret=1
    566 if [ $ret != 0 ]; then echo_i "failed"; fi
    567 status=$((status + ret))
    568 
    569 n=$((n + 1))
    570 echo_i "enable responses from authoritative server ($n)"
    571 ret=0
    572 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
    573 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    574 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
    575 if [ $ret != 0 ]; then echo_i "failed"; fi
    576 status=$((status + ret))
    577 
    578 n=$((n + 1))
    579 echo_i "prime cache longttl.example TXT (low max-stale-ttl) ($n)"
    580 ret=0
    581 $DIG -p ${PORT} @10.53.0.1 longttl.example TXT >dig.out.test$n || ret=1
    582 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    583 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    584 if [ $ret != 0 ]; then echo_i "failed"; fi
    585 status=$((status + ret))
    586 
    587 n=$((n + 1))
    588 echo_i "prime cache data.example TXT (low max-stale-ttl) ($n)"
    589 ret=0
    590 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$n || ret=1
    591 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    592 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    593 if [ $ret != 0 ]; then echo_i "failed"; fi
    594 status=$((status + ret))
    595 
    596 n=$((n + 1))
    597 echo_i "prime cache othertype.example CAA (low max-stale-ttl) ($n)"
    598 ret=0
    599 $DIG -p ${PORT} @10.53.0.1 othertype.example CAA >dig.out.test$n || ret=1
    600 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    601 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    602 if [ $ret != 0 ]; then echo_i "failed"; fi
    603 status=$((status + ret))
    604 
    605 n=$((n + 1))
    606 echo_i "prime cache nodata.example TXT (low max-stale-ttl) ($n)"
    607 ret=0
    608 $DIG -p ${PORT} @10.53.0.1 nodata.example TXT >dig.out.test$n || ret=1
    609 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    610 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    611 if [ $ret != 0 ]; then echo_i "failed"; fi
    612 status=$((status + ret))
    613 
    614 n=$((n + 1))
    615 echo_i "prime cache nxdomain.example TXT (low max-stale-ttl) ($n)"
    616 ret=0
    617 $DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT >dig.out.test$n || ret=1
    618 grep "status: NXDOMAIN" dig.out.test$n >/dev/null || ret=1
    619 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    620 if [ $ret != 0 ]; then echo_i "failed"; fi
    621 status=$((status + ret))
    622 
    623 # Keep track of time so we can access these RRset later, when we expect them
    624 # to become ancient.
    625 t1=$($PERL -e 'print time()')
    626 
    627 n=$((n + 1))
    628 echo_i "verify prime cache statistics (low max-stale-ttl) ($n)"
    629 ret=0
    630 rm -f ns1/named.stats
    631 $RNDCCMD 10.53.0.1 stats >/dev/null 2>&1
    632 [ -f ns1/named.stats ] || ret=1
    633 cp ns1/named.stats ns1/named.stats.$n
    634 # Check first 10 lines of Cache DB statistics.  After prime queries, we expect
    635 # two active TXT RRsets, one active Others, one nxrrset TXT, and one NXDOMAIN.
    636 grep -A 10 "++ Cache DB RRsets ++" ns1/named.stats.$n >ns1/named.stats.$n.cachedb || ret=1
    637 grep "2 TXT" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    638 grep "1 Others" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    639 grep "1 !TXT" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    640 grep "1 NXDOMAIN" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    641 status=$((status + ret))
    642 if [ $ret != 0 ]; then echo_i "failed"; fi
    643 
    644 n=$((n + 1))
    645 echo_i "disable responses from authoritative server ($n)"
    646 ret=0
    647 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
    648 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    649 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
    650 if [ $ret != 0 ]; then echo_i "failed"; fi
    651 status=$((status + ret))
    652 
    653 sleep 2
    654 
    655 echo_i "sending queries for tests $((n + 1))-$((n + 4))..."
    656 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$((n + 1)) &
    657 $DIG -p ${PORT} @10.53.0.1 othertype.example CAA >dig.out.test$((n + 2)) &
    658 $DIG -p ${PORT} @10.53.0.1 nodata.example TXT >dig.out.test$((n + 3)) &
    659 $DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT >dig.out.test$((n + 4)) &
    660 
    661 wait
    662 
    663 n=$((n + 1))
    664 echo_i "check stale data.example TXT (low max-stale-ttl) ($n)"
    665 ret=0
    666 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    667 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    668 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    669 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    670 if [ $ret != 0 ]; then echo_i "failed"; fi
    671 status=$((status + ret))
    672 
    673 n=$((n + 1))
    674 echo_i "check stale othertype.example CAA (low max-stale-ttl) ($n)"
    675 ret=0
    676 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    677 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    678 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    679 grep "othertype\.example\..*3.*IN.*CAA.*0.*issue" dig.out.test$n >/dev/null || ret=1
    680 if [ $ret != 0 ]; then echo_i "failed"; fi
    681 status=$((status + ret))
    682 
    683 n=$((n + 1))
    684 echo_i "check stale nodata.example TXT (low max-stale-ttl) ($n)"
    685 ret=0
    686 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    687 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    688 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    689 grep "example\..*3.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
    690 if [ $ret != 0 ]; then echo_i "failed"; fi
    691 status=$((status + ret))
    692 
    693 n=$((n + 1))
    694 echo_i "check stale nxdomain.example TXT (low max-stale-ttl) ($n)"
    695 ret=0
    696 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    697 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    698 if [ $ret != 0 ]; then echo_i "failed"; fi
    699 status=$((status + ret))
    700 
    701 n=$((n + 1))
    702 echo_i "verify stale cache statistics (low max-stale-ttl) ($n)"
    703 ret=0
    704 rm -f ns1/named.stats
    705 $RNDCCMD 10.53.0.1 stats >/dev/null 2>&1
    706 [ -f ns1/named.stats ] || ret=1
    707 cp ns1/named.stats ns1/named.stats.$n
    708 # Check first 10 lines of Cache DB statistics.  After serve-stale queries, we
    709 # expect one active TXT RRset, one stale TXT, one stale nxrrset TXT, and one
    710 # stale NXDOMAIN.
    711 grep -A 10 "++ Cache DB RRsets ++" ns1/named.stats.$n >ns1/named.stats.$n.cachedb || ret=1
    712 grep "1 TXT" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    713 grep "1 #TXT" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    714 grep "1 #Others" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    715 grep "1 #!TXT" ns1/named.stats.$n.cachedb >/dev/null || ret=1
    716 
    717 status=$((status + ret))
    718 if [ $ret != 0 ]; then echo_i "failed"; fi
    719 
    720 # Retrieve max-stale-ttl value.
    721 interval_to_ancient=$(grep 'max-stale-ttl' ns1/named3.conf.in | awk '{ print $2 }' | tr -d ';')
    722 # We add 2 seconds to it since this is the ttl value of the records being
    723 # tested.
    724 interval_to_ancient=$((interval_to_ancient + 2))
    725 t2=$($PERL -e 'print time()')
    726 elapsed=$((t2 - t1))
    727 
    728 # If elapsed time so far is less than max-stale-ttl + 2 seconds, then we sleep
    729 # enough to ensure that we'll ask for ancient RRsets in the next queries.
    730 if [ $elapsed -lt $interval_to_ancient ]; then
    731   sleep $((interval_to_ancient - elapsed))
    732 fi
    733 
    734 echo_i "sending queries for tests $((n + 1))-$((n + 4))..."
    735 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$((n + 1)) &
    736 $DIG -p ${PORT} @10.53.0.1 othertype.example CAA >dig.out.test$((n + 2)) &
    737 $DIG -p ${PORT} @10.53.0.1 nodata.example TXT >dig.out.test$((n + 3)) &
    738 $DIG -p ${PORT} @10.53.0.1 nxdomain.example TXT >dig.out.test$((n + 4)) &
    739 
    740 wait
    741 
    742 n=$((n + 1))
    743 echo_i "check ancient data.example TXT (low max-stale-ttl) ($n)"
    744 ret=0
    745 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    746 grep "EDE" dig.out.test$n >/dev/null && ret=1
    747 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    748 if [ $ret != 0 ]; then echo_i "failed"; fi
    749 status=$((status + ret))
    750 
    751 n=$((n + 1))
    752 echo_i "check ancient othertype.example CAA (low max-stale-ttl) ($n)"
    753 ret=0
    754 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    755 grep "EDE" dig.out.test$n >/dev/null && ret=1
    756 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    757 if [ $ret != 0 ]; then echo_i "failed"; fi
    758 status=$((status + ret))
    759 
    760 n=$((n + 1))
    761 echo_i "check ancient nodata.example TXT (low max-stale-ttl) ($n)"
    762 ret=0
    763 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    764 grep "EDE" dig.out.test$n >/dev/null && ret=1
    765 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    766 if [ $ret != 0 ]; then echo_i "failed"; fi
    767 status=$((status + ret))
    768 
    769 n=$((n + 1))
    770 echo_i "check ancient nxdomain.example TXT (low max-stale-ttl) ($n)"
    771 ret=0
    772 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
    773 grep "EDE" dig.out.test$n >/dev/null && ret=1
    774 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
    775 if [ $ret != 0 ]; then echo_i "failed"; fi
    776 status=$((status + ret))
    777 
    778 # Test stale-refresh-time when serve-stale is enabled via rndc.
    779 # Steps for testing stale-refresh-time option (default).
    780 # 1. Prime cache data.example txt
    781 # 2. Disable responses from authoritative server.
    782 # 3. Sleep for TTL duration so rrset TTL expires (2 sec)
    783 # 4. Query data.example
    784 # 5. Check if response come from stale rrset (3 sec TTL)
    785 # 6. Enable responses from authoritative server.
    786 # 7. Query data.example
    787 # 8. Check if response come from stale rrset, since the query
    788 #    is within stale-refresh-time window.
    789 n=$((n + 1))
    790 echo_i "flush cache, enable responses from authoritative server ($n)"
    791 ret=0
    792 $RNDCCMD 10.53.0.1 flushtree example >rndc.out.test$n.1 2>&1 || ret=1
    793 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
    794 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    795 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
    796 if [ $ret != 0 ]; then echo_i "failed"; fi
    797 status=$((status + ret))
    798 
    799 n=$((n + 1))
    800 echo_i "check 'rndc serve-stale status' ($n)"
    801 ret=0
    802 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    803 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=3 max-stale-ttl=20 stale-refresh-time=30)' rndc.out.test$n >/dev/null || ret=1
    804 if [ $ret != 0 ]; then echo_i "failed"; fi
    805 status=$((status + ret))
    806 
    807 # Step 1.
    808 n=$((n + 1))
    809 echo_i "prime cache data.example TXT (stale-refresh-time rndc) ($n)"
    810 ret=0
    811 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$n || ret=1
    812 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    813 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    814 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    815 if [ $ret != 0 ]; then echo_i "failed"; fi
    816 status=$((status + ret))
    817 
    818 # Step 2.
    819 n=$((n + 1))
    820 echo_i "disable responses from authoritative server ($n)"
    821 ret=0
    822 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
    823 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    824 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
    825 if [ $ret != 0 ]; then echo_i "failed"; fi
    826 status=$((status + ret))
    827 
    828 # Step 3.
    829 sleep 2
    830 
    831 # Step 4.
    832 n=$((n + 1))
    833 echo_i "sending query for test ($n)"
    834 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$n || ret=1
    835 
    836 # Step 5.
    837 echo_i "check stale data.example TXT (stale-refresh-time rndc) ($n)"
    838 ret=0
    839 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    840 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    841 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    842 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    843 if [ $ret != 0 ]; then echo_i "failed"; fi
    844 status=$((status + ret))
    845 
    846 # Step 6.
    847 n=$((n + 1))
    848 echo_i "enable responses from authoritative server ($n)"
    849 ret=0
    850 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
    851 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    852 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
    853 if [ $ret != 0 ]; then echo_i "failed"; fi
    854 status=$((status + ret))
    855 
    856 # Step 7.
    857 echo_i "sending query for test $((n + 1))"
    858 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$((n + 1)) || true
    859 
    860 # Step 8.
    861 n=$((n + 1))
    862 echo_i "check stale data.example TXT comes from cache (stale-refresh-time rndc) ($n)"
    863 ret=0
    864 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    865 grep "EDE: 3 (Stale Answer): (query within stale refresh time window)" dig.out.test$n >/dev/null || ret=1
    866 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    867 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    868 if [ $ret != 0 ]; then echo_i "failed"; fi
    869 status=$((status + ret))
    870 
    871 # Steps for testing stale-refresh-time option (disabled).
    872 # 1. Prime cache data.example txt
    873 # 2. Disable responses from authoritative server.
    874 # 3. Sleep for TTL duration so rrset TTL expires (2 sec)
    875 # 4. Query data.example
    876 # 5. Check if response come from stale rrset (3 sec TTL)
    877 # 6. Enable responses from authoritative server.
    878 # 7. Query data.example
    879 # 8. Check if response come from stale rrset, since the query
    880 #    is within stale-refresh-time window.
    881 n=$((n + 1))
    882 echo_i "updating ns1/named.conf ($n)"
    883 ret=0
    884 copy_setports ns1/named4.conf.in ns1/named.conf
    885 if [ $ret != 0 ]; then echo_i "failed"; fi
    886 status=$((status + ret))
    887 
    888 n=$((n + 1))
    889 echo_i "running 'rndc reload' ($n)"
    890 ret=0
    891 rndc_reload ns1 10.53.0.1
    892 if [ $ret != 0 ]; then echo_i "failed"; fi
    893 status=$((status + ret))
    894 
    895 n=$((n + 1))
    896 echo_i "check 'rndc serve-stale status' ($n)"
    897 ret=0
    898 $RNDCCMD 10.53.0.1 serve-stale status >rndc.out.test$n 2>&1 || ret=1
    899 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=3 max-stale-ttl=20 stale-refresh-time=0)' rndc.out.test$n >/dev/null || ret=1
    900 if [ $ret != 0 ]; then echo_i "failed"; fi
    901 status=$((status + ret))
    902 
    903 n=$((n + 1))
    904 echo_i "flush cache, enable responses from authoritative server ($n)"
    905 ret=0
    906 $RNDCCMD 10.53.0.1 flushtree example >rndc.out.test$n.1 2>&1 || ret=1
    907 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
    908 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    909 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
    910 if [ $ret != 0 ]; then echo_i "failed"; fi
    911 status=$((status + ret))
    912 
    913 # Step 1.
    914 n=$((n + 1))
    915 echo_i "prime cache data.example TXT (stale-refresh-time disabled) ($n)"
    916 ret=0
    917 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$n || ret=1
    918 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    919 grep "EDE" dig.out.test$n >/dev/null && ret=1
    920 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    921 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    922 if [ $ret != 0 ]; then echo_i "failed"; fi
    923 status=$((status + ret))
    924 
    925 # Step 2.
    926 n=$((n + 1))
    927 echo_i "disable responses from authoritative server ($n)"
    928 ret=0
    929 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
    930 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    931 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
    932 if [ $ret != 0 ]; then echo_i "failed"; fi
    933 status=$((status + ret))
    934 
    935 # Step 3.
    936 sleep 2
    937 
    938 # Step 4.
    939 n=$((n + 1))
    940 echo_i "sending query for test ($n)"
    941 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$n || ret=1
    942 
    943 # Step 5.
    944 echo_i "check stale data.example TXT (stale-refresh-time disabled) ($n)"
    945 ret=0
    946 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    947 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
    948 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    949 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    950 if [ $ret != 0 ]; then echo_i "failed"; fi
    951 status=$((status + ret))
    952 
    953 # Step 6.
    954 n=$((n + 1))
    955 echo_i "enable responses from authoritative server ($n)"
    956 ret=0
    957 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
    958 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    959 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
    960 if [ $ret != 0 ]; then echo_i "failed"; fi
    961 status=$((status + ret))
    962 
    963 # Step 7.
    964 echo_i "sending query for test $((n + 1))"
    965 $DIG -p ${PORT} @10.53.0.1 data.example TXT >dig.out.test$((n + 1)) || true
    966 
    967 # Step 8.
    968 n=$((n + 1))
    969 echo_i "check data.example TXT comes from authoritative (stale-refresh-time disabled) ($n)"
    970 ret=0
    971 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
    972 grep "EDE" dig.out.test$n >/dev/null && ret=1
    973 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    974 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
    975 if [ $ret != 0 ]; then echo_i "failed"; fi
    976 status=$((status + ret))
    977 
    978 #
    979 # Now test server with no serve-stale options set.
    980 #
    981 echo_i "test server with no serve-stale options set"
    982 
    983 n=$((n + 1))
    984 echo_i "updating ns3/named.conf ($n)"
    985 ret=0
    986 copy_setports ns3/named1.conf.in ns3/named.conf
    987 if [ $ret != 0 ]; then echo_i "failed"; fi
    988 status=$((status + ret))
    989 
    990 echo_i "restart ns3"
    991 stop_server --use-rndc --port ${CONTROLPORT} ns3
    992 start_server --noclean --restart --port ${PORT} ns3
    993 
    994 n=$((n + 1))
    995 echo_i "enable responses from authoritative server ($n)"
    996 ret=0
    997 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
    998 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
    999 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   1000 if [ $ret != 0 ]; then echo_i "failed"; fi
   1001 status=$((status + ret))
   1002 
   1003 n=$((n + 1))
   1004 echo_i "prime cache longttl.example TXT (max-stale-ttl default) ($n)"
   1005 ret=0
   1006 $DIG -p ${PORT} @10.53.0.3 longttl.example TXT >dig.out.test$n || ret=1
   1007 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1008 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1009 if [ $ret != 0 ]; then echo_i "failed"; fi
   1010 status=$((status + ret))
   1011 
   1012 n=$((n + 1))
   1013 echo_i "prime cache data.example TXT (max-stale-ttl default) ($n)"
   1014 ret=0
   1015 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   1016 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1017 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1018 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   1019 if [ $ret != 0 ]; then echo_i "failed"; fi
   1020 status=$((status + ret))
   1021 
   1022 n=$((n + 1))
   1023 echo_i "prime cache othertype.example CAA (max-stale-ttl default) ($n)"
   1024 ret=0
   1025 $DIG -p ${PORT} @10.53.0.3 othertype.example CAA >dig.out.test$n || ret=1
   1026 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1027 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1028 grep "othertype\.example\..*2.*IN.*CAA.*0.*issue" dig.out.test$n >/dev/null || ret=1
   1029 if [ $ret != 0 ]; then echo_i "failed"; fi
   1030 status=$((status + ret))
   1031 
   1032 n=$((n + 1))
   1033 echo_i "prime cache nodata.example TXT (max-stale-ttl default) ($n)"
   1034 ret=0
   1035 $DIG -p ${PORT} @10.53.0.3 nodata.example TXT >dig.out.test$n || ret=1
   1036 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1037 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1038 grep "example\..*2.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
   1039 if [ $ret != 0 ]; then echo_i "failed"; fi
   1040 status=$((status + ret))
   1041 
   1042 n=$((n + 1))
   1043 echo_i "prime cache nxdomain.example TXT (max-stale-ttl default) ($n)"
   1044 ret=0
   1045 $DIG -p ${PORT} @10.53.0.3 nxdomain.example TXT >dig.out.test$n || ret=1
   1046 grep "status: NXDOMAIN" dig.out.test$n >/dev/null || ret=1
   1047 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1048 grep "example\..*2.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
   1049 if [ $ret != 0 ]; then echo_i "failed"; fi
   1050 status=$((status + ret))
   1051 
   1052 n=$((n + 1))
   1053 echo_i "verify prime cache statistics (max-stale-ttl default) ($n)"
   1054 ret=0
   1055 rm -f ns3/named.stats
   1056 $RNDCCMD 10.53.0.3 stats >/dev/null 2>&1
   1057 [ -f ns3/named.stats ] || ret=1
   1058 cp ns3/named.stats ns3/named.stats.$n
   1059 # Check first 10 lines of Cache DB statistics.  After prime queries, we expect
   1060 # two active TXT RRsets, one active Others, one nxrrset TXT, and one NXDOMAIN.
   1061 grep -A 10 "++ Cache DB RRsets ++" ns3/named.stats.$n >ns3/named.stats.$n.cachedb || ret=1
   1062 grep "2 TXT" ns3/named.stats.$n.cachedb >/dev/null || ret=1
   1063 grep "1 Others" ns3/named.stats.$n.cachedb >/dev/null || ret=1
   1064 grep "1 !TXT" ns3/named.stats.$n.cachedb >/dev/null || ret=1
   1065 grep "1 NXDOMAIN" ns3/named.stats.$n.cachedb >/dev/null || ret=1
   1066 status=$((status + ret))
   1067 if [ $ret != 0 ]; then echo_i "failed"; fi
   1068 
   1069 n=$((n + 1))
   1070 echo_i "disable responses from authoritative server ($n)"
   1071 ret=0
   1072 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
   1073 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1074 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
   1075 if [ $ret != 0 ]; then echo_i "failed"; fi
   1076 status=$((status + ret))
   1077 
   1078 n=$((n + 1))
   1079 echo_i "check 'rndc serve-stale status' ($n)"
   1080 ret=0
   1081 $RNDCCMD 10.53.0.3 serve-stale status >rndc.out.test$n 2>&1 || ret=1
   1082 grep "_default: stale cache enabled; stale answers disabled (stale-answer-ttl=$stale_answer_ttl max-stale-ttl=$max_stale_ttl stale-refresh-time=30)" rndc.out.test$n >/dev/null || ret=1
   1083 if [ $ret != 0 ]; then echo_i "failed"; fi
   1084 status=$((status + ret))
   1085 
   1086 sleep 2
   1087 
   1088 echo_i "sending queries for tests $((n + 1))-$((n + 4))..."
   1089 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$((n + 1)) &
   1090 $DIG -p ${PORT} @10.53.0.3 othertype.example CAA >dig.out.test$((n + 2)) &
   1091 $DIG -p ${PORT} @10.53.0.3 nodata.example TXT >dig.out.test$((n + 3)) &
   1092 $DIG -p ${PORT} @10.53.0.3 nxdomain.example TXT >dig.out.test$((n + 4)) &
   1093 
   1094 wait
   1095 
   1096 n=$((n + 1))
   1097 echo_i "check fail of data.example TXT (max-stale-ttl default) ($n)"
   1098 ret=0
   1099 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1100 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1101 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1102 if [ $ret != 0 ]; then echo_i "failed"; fi
   1103 status=$((status + ret))
   1104 
   1105 n=$((n + 1))
   1106 echo_i "check fail of othertype.example CAA (max-stale-ttl default) ($n)"
   1107 ret=0
   1108 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1109 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1110 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1111 if [ $ret != 0 ]; then echo_i "failed"; fi
   1112 status=$((status + ret))
   1113 
   1114 n=$((n + 1))
   1115 echo_i "check fail of nodata.example TXT (max-stale-ttl default) ($n)"
   1116 ret=0
   1117 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1118 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1119 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1120 if [ $ret != 0 ]; then echo_i "failed"; fi
   1121 status=$((status + ret))
   1122 
   1123 n=$((n + 1))
   1124 echo_i "check fail of nxdomain.example TXT (max-stale-ttl default) ($n)"
   1125 ret=0
   1126 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1127 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1128 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1129 if [ $ret != 0 ]; then echo_i "failed"; fi
   1130 status=$((status + ret))
   1131 
   1132 n=$((n + 1))
   1133 echo_i "verify stale cache statistics (max-stale-ttl default) ($n)"
   1134 ret=0
   1135 rm -f ns3/named.stats
   1136 $RNDCCMD 10.53.0.3 stats >/dev/null 2>&1
   1137 [ -f ns3/named.stats ] || ret=1
   1138 cp ns3/named.stats ns3/named.stats.$n
   1139 # Check first 10 lines of Cache DB statistics. After last queries, we expect
   1140 # one active TXT RRset, one stale TXT, one stale nxrrset TXT, and one stale
   1141 # NXDOMAIN.
   1142 grep -A 10 "++ Cache DB RRsets ++" ns3/named.stats.$n >ns3/named.stats.$n.cachedb || ret=1
   1143 grep "1 TXT" ns3/named.stats.$n.cachedb >/dev/null || ret=1
   1144 grep "1 #TXT" ns3/named.stats.$n.cachedb >/dev/null || ret=1
   1145 grep "1 #Others" ns3/named.stats.$n.cachedb >/dev/null || ret=1
   1146 grep "1 #!TXT" ns3/named.stats.$n.cachedb >/dev/null || ret=1
   1147 
   1148 status=$((status + ret))
   1149 if [ $ret != 0 ]; then echo_i "failed"; fi
   1150 
   1151 n=$((n + 1))
   1152 echo_i "check 'rndc serve-stale on' ($n)"
   1153 ret=0
   1154 $RNDCCMD 10.53.0.3 serve-stale on >rndc.out.test$n 2>&1 || ret=1
   1155 if [ $ret != 0 ]; then echo_i "failed"; fi
   1156 status=$((status + ret))
   1157 
   1158 n=$((n + 1))
   1159 echo_i "check 'rndc serve-stale status' ($n)"
   1160 ret=0
   1161 $RNDCCMD 10.53.0.3 serve-stale status >rndc.out.test$n 2>&1 || ret=1
   1162 grep "_default: stale cache enabled; stale answers enabled (stale-answer-ttl=$stale_answer_ttl max-stale-ttl=$max_stale_ttl stale-refresh-time=30)" rndc.out.test$n >/dev/null || ret=1
   1163 if [ $ret != 0 ]; then echo_i "failed"; fi
   1164 status=$((status + ret))
   1165 
   1166 sleep 2
   1167 
   1168 # Check that if we don't have stale data for a domain name, we will
   1169 # not answer anything until the resolver query timeout.
   1170 n=$((n + 1))
   1171 echo_i "check notincache.example TXT times out (max-stale-ttl default) ($n)"
   1172 ret=0
   1173 $DIG -p ${PORT} +tries=1 +timeout=3 @10.53.0.3 notfound.example TXT >dig.out.test$n 2>&1 && ret=1
   1174 grep "timed out" dig.out.test$n >/dev/null || ret=1
   1175 grep ";; no servers could be reached" dig.out.test$n >/dev/null || ret=1
   1176 if [ $ret != 0 ]; then echo_i "failed"; fi
   1177 status=$((status + ret))
   1178 
   1179 echo_i "sending queries for tests $((n + 1))-$((n + 4))..."
   1180 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$((n + 1)) &
   1181 $DIG -p ${PORT} @10.53.0.3 othertype.example CAA >dig.out.test$((n + 2)) &
   1182 $DIG -p ${PORT} @10.53.0.3 nodata.example TXT >dig.out.test$((n + 3)) &
   1183 $DIG -p ${PORT} @10.53.0.3 nxdomain.example TXT >dig.out.test$((n + 4)) &
   1184 $DIG -p ${PORT} @10.53.0.3 notfound.example TXT >dig.out.test$((n + 5)) &
   1185 
   1186 wait
   1187 
   1188 n=$((n + 1))
   1189 echo_i "check data.example TXT (max-stale-ttl default) ($n)"
   1190 ret=0
   1191 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1192 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
   1193 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1194 grep "data\.example\..*30.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   1195 if [ $ret != 0 ]; then echo_i "failed"; fi
   1196 status=$((status + ret))
   1197 
   1198 n=$((n + 1))
   1199 echo_i "check othertype.example CAA (max-stale-ttl default) ($n)"
   1200 ret=0
   1201 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1202 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
   1203 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1204 grep "example\..*30.*IN.*CAA.*0.*issue" dig.out.test$n >/dev/null || ret=1
   1205 if [ $ret != 0 ]; then echo_i "failed"; fi
   1206 status=$((status + ret))
   1207 
   1208 n=$((n + 1))
   1209 echo_i "check nodata.example TXT (max-stale-ttl default) ($n)"
   1210 ret=0
   1211 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1212 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
   1213 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1214 grep "example\..*30.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
   1215 if [ $ret != 0 ]; then echo_i "failed"; fi
   1216 status=$((status + ret))
   1217 
   1218 n=$((n + 1))
   1219 echo_i "check nxdomain.example TXT (max-stale-ttl default) ($n)"
   1220 ret=0
   1221 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1222 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1223 if [ $ret != 0 ]; then echo_i "failed"; fi
   1224 status=$((status + ret))
   1225 
   1226 # The notfound.example check is different than nxdomain.example because
   1227 # we didn't send a prime query to add notfound.example to the cache.
   1228 n=$((n + 1))
   1229 echo_i "check notfound.example TXT (max-stale-ttl default) ($n)"
   1230 ret=0
   1231 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1232 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1233 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1234 if [ $ret != 0 ]; then echo_i "failed"; fi
   1235 status=$((status + ret))
   1236 
   1237 #
   1238 # Now test server with serve-stale answers disabled.
   1239 #
   1240 echo_i "test server with serve-stale disabled"
   1241 
   1242 n=$((n + 1))
   1243 echo_i "enable responses from authoritative server ($n)"
   1244 ret=0
   1245 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
   1246 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1247 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   1248 if [ $ret != 0 ]; then echo_i "failed"; fi
   1249 status=$((status + ret))
   1250 
   1251 n=$((n + 1))
   1252 echo_i "prime cache longttl.example TTL (serve-stale answers disabled) ($n)"
   1253 ret=0
   1254 $DIG -p ${PORT} @10.53.0.4 longttl.example TXT >dig.out.test$n || ret=1
   1255 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1256 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1257 if [ $ret != 0 ]; then echo_i "failed"; fi
   1258 status=$((status + ret))
   1259 
   1260 n=$((n + 1))
   1261 echo_i "prime cache data.example TTL (serve-stale answers disabled) ($n)"
   1262 ret=0
   1263 $DIG -p ${PORT} @10.53.0.4 data.example TXT >dig.out.test$n || ret=1
   1264 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1265 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1266 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   1267 if [ $ret != 0 ]; then echo_i "failed"; fi
   1268 status=$((status + ret))
   1269 
   1270 n=$((n + 1))
   1271 echo_i "prime cache othertype.example CAA (serve-stale answers disabled) ($n)"
   1272 ret=0
   1273 $DIG -p ${PORT} @10.53.0.4 othertype.example CAA >dig.out.test$n || ret=1
   1274 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1275 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1276 grep "othertype\.example\..*2.*IN.*CAA.*0.*issue" dig.out.test$n >/dev/null || ret=1
   1277 if [ $ret != 0 ]; then echo_i "failed"; fi
   1278 status=$((status + ret))
   1279 
   1280 n=$((n + 1))
   1281 echo_i "prime cache nodata.example TXT (serve-stale answers disabled) ($n)"
   1282 ret=0
   1283 $DIG -p ${PORT} @10.53.0.4 nodata.example TXT >dig.out.test$n || ret=1
   1284 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1285 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1286 grep "example\..*2.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
   1287 if [ $ret != 0 ]; then echo_i "failed"; fi
   1288 status=$((status + ret))
   1289 
   1290 n=$((n + 1))
   1291 echo_i "prime cache nxdomain.example TXT (serve-stale answers disabled) ($n)"
   1292 ret=0
   1293 $DIG -p ${PORT} @10.53.0.4 nxdomain.example TXT >dig.out.test$n || ret=1
   1294 grep "status: NXDOMAIN" dig.out.test$n >/dev/null || ret=1
   1295 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1296 grep "example\..*2.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
   1297 if [ $ret != 0 ]; then echo_i "failed"; fi
   1298 status=$((status + ret))
   1299 
   1300 n=$((n + 1))
   1301 echo_i "verify prime cache statistics (serve-stale answers disabled) ($n)"
   1302 ret=0
   1303 rm -f ns4/named.stats
   1304 $RNDCCMD 10.53.0.4 stats >/dev/null 2>&1
   1305 [ -f ns4/named.stats ] || ret=1
   1306 cp ns4/named.stats ns4/named.stats.$n
   1307 # Check first 10 lines of Cache DB statistics.  After prime queries, we expect
   1308 # two active TXT RRsets, one active Others, one nxrrset TXT, and one NXDOMAIN.
   1309 grep -A 10 "++ Cache DB RRsets ++" ns4/named.stats.$n >ns4/named.stats.$n.cachedb || ret=1
   1310 grep "2 TXT" ns4/named.stats.$n.cachedb >/dev/null || ret=1
   1311 grep "1 Others" ns4/named.stats.$n.cachedb >/dev/null || ret=1
   1312 grep "1 !TXT" ns4/named.stats.$n.cachedb >/dev/null || ret=1
   1313 grep "1 NXDOMAIN" ns4/named.stats.$n.cachedb >/dev/null || ret=1
   1314 status=$((status + ret))
   1315 if [ $ret != 0 ]; then echo_i "failed"; fi
   1316 
   1317 n=$((n + 1))
   1318 echo_i "disable responses from authoritative server ($n)"
   1319 ret=0
   1320 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
   1321 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1322 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
   1323 if [ $ret != 0 ]; then echo_i "failed"; fi
   1324 status=$((status + ret))
   1325 
   1326 n=$((n + 1))
   1327 echo_i "check 'rndc serve-stale status' ($n)"
   1328 ret=0
   1329 $RNDCCMD 10.53.0.4 serve-stale status >rndc.out.test$n 2>&1 || ret=1
   1330 grep "_default: stale cache enabled; stale answers disabled (stale-answer-ttl=$stale_answer_ttl max-stale-ttl=$max_stale_ttl stale-refresh-time=30)" rndc.out.test$n >/dev/null || ret=1
   1331 if [ $ret != 0 ]; then echo_i "failed"; fi
   1332 status=$((status + ret))
   1333 
   1334 sleep 2
   1335 
   1336 echo_i "sending queries for tests $((n + 1))-$((n + 4))..."
   1337 $DIG -p ${PORT} @10.53.0.4 data.example TXT >dig.out.test$((n + 1)) &
   1338 $DIG -p ${PORT} @10.53.0.4 othertype.example CAA >dig.out.test$((n + 2)) &
   1339 $DIG -p ${PORT} @10.53.0.4 nodata.example TXT >dig.out.test$((n + 3)) &
   1340 $DIG -p ${PORT} @10.53.0.4 nxdomain.example TXT >dig.out.test$((n + 4)) &
   1341 
   1342 wait
   1343 
   1344 n=$((n + 1))
   1345 echo_i "check fail of data.example TXT (serve-stale answers disabled) ($n)"
   1346 ret=0
   1347 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1348 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1349 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1350 if [ $ret != 0 ]; then echo_i "failed"; fi
   1351 status=$((status + ret))
   1352 
   1353 n=$((n + 1))
   1354 echo_i "check fail of othertype.example TXT (serve-stale answers disabled) ($n)"
   1355 ret=0
   1356 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1357 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1358 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1359 if [ $ret != 0 ]; then echo_i "failed"; fi
   1360 status=$((status + ret))
   1361 
   1362 n=$((n + 1))
   1363 echo_i "check fail of nodata.example TXT (serve-stale answers disabled) ($n)"
   1364 ret=0
   1365 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1366 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1367 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1368 if [ $ret != 0 ]; then echo_i "failed"; fi
   1369 status=$((status + ret))
   1370 
   1371 n=$((n + 1))
   1372 echo_i "check fail of nxdomain.example TXT (serve-stale answers disabled) ($n)"
   1373 ret=0
   1374 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1375 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1376 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1377 if [ $ret != 0 ]; then echo_i "failed"; fi
   1378 status=$((status + ret))
   1379 
   1380 n=$((n + 1))
   1381 echo_i "verify stale cache statistics (serve-stale answers disabled) ($n)"
   1382 ret=0
   1383 rm -f ns4/named.stats
   1384 $RNDCCMD 10.53.0.4 stats >/dev/null 2>&1
   1385 [ -f ns4/named.stats ] || ret=1
   1386 cp ns4/named.stats ns4/named.stats.$n
   1387 # Check first 10 lines of Cache DB statistics. After last queries, we expect
   1388 # one active TXT RRset, one stale TXT, one stale nxrrset TXT, and one stale
   1389 # NXDOMAIN.
   1390 grep -A 10 "++ Cache DB RRsets ++" ns4/named.stats.$n >ns4/named.stats.$n.cachedb || ret=1
   1391 grep "1 TXT" ns4/named.stats.$n.cachedb >/dev/null || ret=1
   1392 grep "1 #TXT" ns4/named.stats.$n.cachedb >/dev/null || ret=1
   1393 grep "1 #Others" ns4/named.stats.$n.cachedb >/dev/null || ret=1
   1394 grep "1 #!TXT" ns4/named.stats.$n.cachedb >/dev/null || ret=1
   1395 status=$((status + ret))
   1396 if [ $ret != 0 ]; then echo_i "failed"; fi
   1397 
   1398 # Dump the cache.
   1399 n=$((n + 1))
   1400 echo_i "dump the cache (serve-stale answers disabled) ($n)"
   1401 ret=0
   1402 rndc_dumpdb ns4 -cache || ret=1
   1403 if [ $ret != 0 ]; then echo_i "failed"; fi
   1404 status=$((status + ret))
   1405 
   1406 echo_i "stop ns4"
   1407 stop_server --use-rndc --port ${CONTROLPORT} ns4
   1408 
   1409 # Load the cache as if it was five minutes (RBTDB_VIRTUAL) older. Since
   1410 # max-stale-ttl defaults to a week, we need to adjust the date by one week and
   1411 # five minutes.
   1412 LASTWEEK=$(TZ=UTC perl -e 'my $now = time();
   1413         my $oneWeekAgo = $now - 604800;
   1414         my $fiveMinutesAgo = $oneWeekAgo - 300;
   1415         my ($s, $m, $h, $d, $mo, $y) = (localtime($fiveMinutesAgo))[0, 1, 2, 3, 4, 5];
   1416         printf("%04d%02d%02d%02d%02d%02d", $y+1900, $mo+1, $d, $h, $m, $s);')
   1417 
   1418 echo_i "mock the cache date to $LASTWEEK (serve-stale answers disabled) ($n)"
   1419 ret=0
   1420 sed -E "s/DATE [0-9]{14}/DATE $LASTWEEK/g" ns4/named_dump.db.test$n >ns4/named_dump.db.out || ret=1
   1421 cp ns4/named_dump.db.out ns4/named_dump.db
   1422 if [ $ret != 0 ]; then echo_i "failed"; fi
   1423 status=$((status + ret))
   1424 
   1425 echo_i "start ns4"
   1426 start_server --noclean --restart --port ${PORT} ns4
   1427 
   1428 n=$((n + 1))
   1429 echo_i "verify ancient cache statistics (serve-stale answers disabled) ($n)"
   1430 ret=0
   1431 rm -f ns4/named.stats
   1432 $RNDCCMD 10.53.0.4 stats #> /dev/null 2>&1
   1433 [ -f ns4/named.stats ] || ret=1
   1434 cp ns4/named.stats ns4/named.stats.$n
   1435 # Check first 10 lines of Cache DB statistics. After last queries, we expect
   1436 # everything to be removed or scheduled to be removed.
   1437 grep -A 10 "++ Cache DB RRsets ++" ns4/named.stats.$n >ns4/named.stats.$n.cachedb || ret=1
   1438 grep "#TXT" ns4/named.stats.$n.cachedb >/dev/null && ret=1
   1439 grep "#Others" ns4/named.stats.$n.cachedb >/dev/null && ret=1
   1440 grep "#!TXT" ns4/named.stats.$n.cachedb >/dev/null && ret=1
   1441 grep "#NXDOMAIN" ns4/named.stats.$n.cachedb >/dev/null && ret=1
   1442 status=$((status + ret))
   1443 if [ $ret != 0 ]; then echo_i "failed"; fi
   1444 
   1445 #
   1446 # Test the server with stale-cache disabled.
   1447 #
   1448 echo_i "test server with serve-stale cache disabled"
   1449 
   1450 n=$((n + 1))
   1451 echo_i "enable responses from authoritative server ($n)"
   1452 ret=0
   1453 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
   1454 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1455 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   1456 if [ $ret != 0 ]; then echo_i "failed"; fi
   1457 status=$((status + ret))
   1458 
   1459 n=$((n + 1))
   1460 echo_i "prime cache longttl.example TXT (serve-stale cache disabled) ($n)"
   1461 ret=0
   1462 $DIG -p ${PORT} @10.53.0.5 longttl.example TXT >dig.out.test$n || ret=1
   1463 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1464 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1465 if [ $ret != 0 ]; then echo_i "failed"; fi
   1466 status=$((status + ret))
   1467 
   1468 n=$((n + 1))
   1469 echo_i "prime cache data.example TXT (serve-stale cache disabled) ($n)"
   1470 ret=0
   1471 $DIG -p ${PORT} @10.53.0.5 data.example TXT >dig.out.test$n || ret=1
   1472 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1473 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1474 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   1475 if [ $ret != 0 ]; then echo_i "failed"; fi
   1476 status=$((status + ret))
   1477 
   1478 n=$((n + 1))
   1479 echo_i "prime cache othertype.example CAA (serve-stale cache disabled) ($n)"
   1480 ret=0
   1481 $DIG -p ${PORT} @10.53.0.5 othertype.example CAA >dig.out.test$n || ret=1
   1482 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1483 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1484 grep "othertype\.example\..*2.*IN.*CAA.*0.*issue" dig.out.test$n >/dev/null || ret=1
   1485 if [ $ret != 0 ]; then echo_i "failed"; fi
   1486 status=$((status + ret))
   1487 
   1488 n=$((n + 1))
   1489 echo_i "prime cache nodata.example TXT (serve-stale cache disabled) ($n)"
   1490 ret=0
   1491 $DIG -p ${PORT} @10.53.0.5 nodata.example TXT >dig.out.test$n || ret=1
   1492 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1493 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1494 grep "example\..*2.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
   1495 if [ $ret != 0 ]; then echo_i "failed"; fi
   1496 status=$((status + ret))
   1497 
   1498 n=$((n + 1))
   1499 echo_i "prime cache nxdomain.example TXT (serve-stale cache disabled) ($n)"
   1500 ret=0
   1501 $DIG -p ${PORT} @10.53.0.5 nxdomain.example TXT >dig.out.test$n || ret=1
   1502 grep "status: NXDOMAIN" dig.out.test$n >/dev/null || ret=1
   1503 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1504 grep "example\..*2.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
   1505 if [ $ret != 0 ]; then echo_i "failed"; fi
   1506 status=$((status + ret))
   1507 
   1508 n=$((n + 1))
   1509 echo_i "verify prime cache statistics (serve-stale cache disabled) ($n)"
   1510 ret=0
   1511 rm -f ns5/named.stats
   1512 $RNDCCMD 10.53.0.5 stats >/dev/null 2>&1
   1513 [ -f ns5/named.stats ] || ret=1
   1514 cp ns5/named.stats ns5/named.stats.$n
   1515 # Check first 10 lines of Cache DB statistics.  After serve-stale queries,
   1516 # we expect two active TXT RRsets, one active Others, one nxrrset TXT, and
   1517 # one NXDOMAIN.
   1518 grep -A 10 "++ Cache DB RRsets ++" ns5/named.stats.$n >ns5/named.stats.$n.cachedb || ret=1
   1519 grep "2 TXT" ns5/named.stats.$n.cachedb >/dev/null || ret=1
   1520 grep "1 Others" ns5/named.stats.$n.cachedb >/dev/null || ret=1
   1521 grep "1 !TXT" ns5/named.stats.$n.cachedb >/dev/null || ret=1
   1522 status=$((status + ret))
   1523 if [ $ret != 0 ]; then echo_i "failed"; fi
   1524 
   1525 n=$((n + 1))
   1526 echo_i "disable responses from authoritative server ($n)"
   1527 ret=0
   1528 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
   1529 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1530 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
   1531 if [ $ret != 0 ]; then echo_i "failed"; fi
   1532 status=$((status + ret))
   1533 
   1534 n=$((n + 1))
   1535 echo_i "check 'rndc serve-stale status' ($n)"
   1536 ret=0
   1537 $RNDCCMD 10.53.0.5 serve-stale status >rndc.out.test$n 2>&1 || ret=1
   1538 grep "_default: stale cache disabled; stale answers unavailable" rndc.out.test$n >/dev/null || ret=1
   1539 if [ $ret != 0 ]; then echo_i "failed"; fi
   1540 status=$((status + ret))
   1541 
   1542 sleep 2
   1543 
   1544 echo_i "sending queries for tests $((n + 1))-$((n + 4))..."
   1545 $DIG -p ${PORT} @10.53.0.5 data.example TXT >dig.out.test$((n + 1)) &
   1546 $DIG -p ${PORT} @10.53.0.5 othertype.example CAA >dig.out.test$((n + 2)) &
   1547 $DIG -p ${PORT} @10.53.0.5 nodata.example TXT >dig.out.test$((n + 3)) &
   1548 $DIG -p ${PORT} @10.53.0.5 nxdomain.example TXT >dig.out.test$((n + 4)) &
   1549 
   1550 wait
   1551 
   1552 n=$((n + 1))
   1553 echo_i "check fail of data.example TXT (serve-stale cache disabled) ($n)"
   1554 ret=0
   1555 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1556 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1557 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1558 if [ $ret != 0 ]; then echo_i "failed"; fi
   1559 status=$((status + ret))
   1560 
   1561 n=$((n + 1))
   1562 echo_i "check fail of othertype.example CAA (serve-stale cache disabled) ($n)"
   1563 ret=0
   1564 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1565 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1566 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1567 if [ $ret != 0 ]; then echo_i "failed"; fi
   1568 status=$((status + ret))
   1569 
   1570 n=$((n + 1))
   1571 echo_i "check fail of nodata.example TXT (serve-stale cache disabled) ($n)"
   1572 ret=0
   1573 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1574 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1575 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1576 if [ $ret != 0 ]; then echo_i "failed"; fi
   1577 status=$((status + ret))
   1578 
   1579 n=$((n + 1))
   1580 echo_i "check fail of nxdomain.example TXT (serve-stale cache disabled) ($n)"
   1581 ret=0
   1582 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   1583 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1584 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1585 if [ $ret != 0 ]; then echo_i "failed"; fi
   1586 status=$((status + ret))
   1587 
   1588 n=$((n + 1))
   1589 echo_i "verify stale cache statistics (serve-stale cache disabled) ($n)"
   1590 ret=0
   1591 rm -f ns5/named.stats
   1592 $RNDCCMD 10.53.0.5 stats >/dev/null 2>&1
   1593 [ -f ns5/named.stats ] || ret=1
   1594 cp ns5/named.stats ns5/named.stats.$n
   1595 # Check first 10 lines of Cache DB statistics.  After serve-stale queries,
   1596 # we expect one active TXT (longttl) and the rest to be expired from cache,
   1597 # but since we keep everything for 5 minutes (RBTDB_VIRTUAL) in the cache
   1598 # after expiry, they still show up in the stats.
   1599 grep -A 10 "++ Cache DB RRsets ++" ns5/named.stats.$n >ns5/named.stats.$n.cachedb || ret=1
   1600 grep -F "1 Others" ns5/named.stats.$n.cachedb >/dev/null || ret=1
   1601 grep -F "2 TXT" ns5/named.stats.$n.cachedb >/dev/null || ret=1
   1602 grep -F "1 !TXT" ns5/named.stats.$n.cachedb >/dev/null || ret=1
   1603 status=$((status + ret))
   1604 if [ $ret != 0 ]; then echo_i "failed"; fi
   1605 
   1606 # Dump the cache.
   1607 n=$((n + 1))
   1608 echo_i "dump the cache (serve-stale cache disabled) ($n)"
   1609 ret=0
   1610 rndc_dumpdb ns5 || ret=1
   1611 if [ $ret != 0 ]; then echo_i "failed"; fi
   1612 status=$((status + ret))
   1613 # Check that expired records are not dumped.
   1614 ret=0
   1615 grep "; expired since .* (awaiting cleanup)" ns5/named_dump.db.test$n && ret=1
   1616 if [ $ret != 0 ]; then echo_i "failed"; fi
   1617 status=$((status + ret))
   1618 
   1619 # Dump the cache including expired entries.
   1620 n=$((n + 1))
   1621 echo_i "dump the cache including expired entries (serve-stale cache disabled) ($n)"
   1622 ret=0
   1623 rndc_dumpdb ns5 -expired || ret=1
   1624 if [ $ret != 0 ]; then echo_i "failed"; fi
   1625 status=$((status + ret))
   1626 
   1627 # Check that expired records are dumped.
   1628 echo_i "check rndc dump expired data.example ($n)"
   1629 ret=0
   1630 awk '/; expired/ { x=$0; getline; print x, $0}' ns5/named_dump.db.test$n \
   1631   | grep "; expired since .* (awaiting cleanup) data\.example\..*A text record with a 2 second ttl" >/dev/null 2>&1 || ret=1
   1632 awk '/; expired/ { x=$0; getline; print x, $0}' ns5/named_dump.db.test$n \
   1633   | grep "; expired since .* (awaiting cleanup) nodata\.example\." >/dev/null 2>&1 || ret=1
   1634 awk '/; expired/ { x=$0; getline; print x, $0}' ns5/named_dump.db.test$n \
   1635   | grep "; expired since .* (awaiting cleanup) nxdomain\.example\." >/dev/null 2>&1 || ret=1
   1636 awk '/; expired/ { x=$0; getline; print x, $0}' ns5/named_dump.db.test$n \
   1637   | grep "; expired since .* (awaiting cleanup) othertype\.example\." >/dev/null 2>&1 || ret=1
   1638 # Also make sure the not expired data does not have an expired comment.
   1639 awk '/; authanswer/ { x=$0; getline; print x, $0}' ns5/named_dump.db.test$n \
   1640   | grep "; authanswer longttl\.example.*A text record with a 600 second ttl" >/dev/null 2>&1 || ret=1
   1641 if [ $ret != 0 ]; then echo_i "failed"; fi
   1642 status=$((status + ret))
   1643 
   1644 echo_i "stop ns5"
   1645 stop_server --use-rndc --port ${CONTROLPORT} ns5
   1646 
   1647 # Load the cache as if it was five minutes (RBTDB_VIRTUAL) older.
   1648 cp ns5/named_dump.db.test$n ns5/named_dump.db
   1649 FIVEMINUTESAGO=$(TZ=UTC perl -e 'my $now = time();
   1650         my $fiveMinutesAgo = 300;
   1651         my ($s, $m, $h, $d, $mo, $y) = (localtime($fiveMinutesAgo))[0, 1, 2, 3, 4, 5];
   1652         printf("%04d%02d%02d%02d%02d%02d", $y+1900, $mo+1, $d, $h, $m, $s);')
   1653 
   1654 n=$((n + 1))
   1655 echo_i "mock the cache date to $FIVEMINUTESAGO (serve-stale cache disabled) ($n)"
   1656 ret=0
   1657 sed -E "s/DATE [0-9]{14}/DATE $FIVEMINUTESAGO/g" ns5/named_dump.db >ns5/named_dump.db.out || ret=1
   1658 cp ns5/named_dump.db.out ns5/named_dump.db
   1659 if [ $ret != 0 ]; then echo_i "failed"; fi
   1660 status=$((status + ret))
   1661 
   1662 echo_i "start ns5"
   1663 start_server --noclean --restart --port ${PORT} ns5
   1664 
   1665 n=$((n + 1))
   1666 echo_i "verify ancient cache statistics (serve-stale cache disabled) ($n)"
   1667 ret=0
   1668 rm -f ns5/named.stats
   1669 $RNDCCMD 10.53.0.5 stats #> /dev/null 2>&1
   1670 [ -f ns5/named.stats ] || ret=1
   1671 cp ns5/named.stats ns5/named.stats.$n
   1672 # Check first 10 lines of Cache DB statistics. After last queries, we expect
   1673 # everything to be removed or scheduled to be removed.
   1674 grep -A 10 "++ Cache DB RRsets ++" ns5/named.stats.$n >ns5/named.stats.$n.cachedb || ret=1
   1675 grep -F "#TXT" ns5/named.stats.$n.cachedb >/dev/null && ret=1
   1676 grep -F "#Others" ns5/named.stats.$n.cachedb >/dev/null && ret=1
   1677 grep -F "#!TXT" ns5/named.stats.$n.cachedb >/dev/null && ret=1
   1678 status=$((status + ret))
   1679 if [ $ret != 0 ]; then echo_i "failed"; fi
   1680 
   1681 ################################################
   1682 # Test for stale-answer-client-timeout (1.8s). #
   1683 ################################################
   1684 echo_i "test stale-answer-client-timeout (1.8)"
   1685 
   1686 n=$((n + 1))
   1687 echo_i "updating ns3/named.conf ($n)"
   1688 ret=0
   1689 copy_setports ns3/named2.conf.in ns3/named.conf
   1690 if [ $ret != 0 ]; then echo_i "failed"; fi
   1691 status=$((status + ret))
   1692 
   1693 echo_i "restart ns3"
   1694 stop_server --use-rndc --port ${CONTROLPORT} ns3
   1695 start_server --noclean --restart --port ${PORT} ns3
   1696 
   1697 n=$((n + 1))
   1698 echo_i "check 'rndc serve-stale status' ($n)"
   1699 ret=0
   1700 $RNDCCMD 10.53.0.3 serve-stale status >rndc.out.test$n 2>&1 || ret=1
   1701 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=3 max-stale-ttl=3600 stale-refresh-time=0)' rndc.out.test$n >/dev/null || ret=1
   1702 if [ $ret != 0 ]; then echo_i "failed"; fi
   1703 status=$((status + ret))
   1704 
   1705 n=$((n + 1))
   1706 echo_i "enable responses from authoritative server ($n)"
   1707 ret=0
   1708 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
   1709 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1710 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   1711 if [ $ret != 0 ]; then echo_i "failed"; fi
   1712 status=$((status + ret))
   1713 
   1714 n=$((n + 1))
   1715 echo_i "prime cache data.example TXT (stale-answer-client-timeout) ($n)"
   1716 ret=0
   1717 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   1718 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1719 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1720 if [ $ret != 0 ]; then echo_i "failed"; fi
   1721 status=$((status + ret))
   1722 
   1723 n=$((n + 1))
   1724 echo_i "prime cache nodata.example TXT (stale-answer-client-timeout) ($n)"
   1725 ret=0
   1726 $DIG -p ${PORT} @10.53.0.3 nodata.example TXT >dig.out.test$n || ret=1
   1727 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1728 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1729 if [ $ret != 0 ]; then echo_i "failed"; fi
   1730 status=$((status + ret))
   1731 
   1732 n=$((n + 1))
   1733 echo_i "delay responses from authoritative server ($n)"
   1734 ret=0
   1735 $DIG -p ${PORT} @10.53.0.2 txt slowdown >dig.out.test$n || ret=1
   1736 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1737 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   1738 if [ $ret != 0 ]; then echo_i "failed"; fi
   1739 status=$((status + ret))
   1740 
   1741 n=$((n + 1))
   1742 echo_i "prime cache data.slow TXT (stale-answer-client-timeout) ($n)"
   1743 ret=0
   1744 $DIG -p ${PORT} @10.53.0.3 data.slow TXT >dig.out.test$n || ret=1
   1745 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1746 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1747 if [ $ret != 0 ]; then echo_i "failed"; fi
   1748 status=$((status + ret))
   1749 
   1750 n=$((n + 1))
   1751 echo_i "disable responses from authoritative server ($n)"
   1752 ret=0
   1753 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
   1754 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1755 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
   1756 if [ $ret != 0 ]; then echo_i "failed"; fi
   1757 status=$((status + ret))
   1758 
   1759 # Allow RRset to become stale.
   1760 sleep 2
   1761 
   1762 nextpart ns3/named.run >/dev/null
   1763 
   1764 echo_i "sending queries for tests $((n + 1))-$((n + 3))..."
   1765 t1=$($PERL -e 'print time()')
   1766 $DIG -p ${PORT} +tries=1 +timeout=11 @10.53.0.3 data.example TXT >dig.out.test$((n + 1)) &
   1767 $DIG -p ${PORT} +tries=1 +timeout=11 @10.53.0.3 nodata.example TXT >dig.out.test$((n + 2)) &
   1768 $DIG -p ${PORT} +tries=1 +timeout=11 @10.53.0.3 data.slow TXT >dig.out.test$((n + 3)) &
   1769 wait
   1770 t2=$($PERL -e 'print time()')
   1771 
   1772 # We configured a long value of 30 seconds for resolver-query-timeout.
   1773 # That should give us enough time to receive an stale answer from cache
   1774 # after stale-answer-client-timeout timer of 1.8 sec triggers.
   1775 n=$((n + 1))
   1776 echo_i "check stale data.example TXT comes from cache (stale-answer-client-timeout 1.8) ($n)"
   1777 ret=0
   1778 wait_for_log 5 "data.example client timeout, stale answer used" ns3/named.run || ret=1
   1779 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1780 grep "EDE: 3 (Stale Answer): (client timeout)" dig.out.test$n >/dev/null || ret=1
   1781 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1782 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   1783 # Configured stale-answer-client-timeout is 1.8s, we allow some extra time
   1784 # just in case other tests are taking too much cpu.
   1785 [ $((t2 - t1)) -le 10 ] || {
   1786   echo_i "query took $((t2 - t1))s to resolve."
   1787   ret=1
   1788 }
   1789 if [ $ret != 0 ]; then echo_i "failed"; fi
   1790 status=$((status + ret))
   1791 
   1792 n=$((n + 1))
   1793 echo_i "check stale nodata.example TXT comes from cache (stale-answer-client-timeout 1.8) ($n)"
   1794 ret=0
   1795 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1796 grep "EDE: 3 (Stale Answer): (client timeout)" dig.out.test$n >/dev/null || ret=1
   1797 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   1798 grep "example\..*3.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
   1799 if [ $ret != 0 ]; then echo_i "failed"; fi
   1800 status=$((status + ret))
   1801 
   1802 n=$((n + 1))
   1803 echo_i "check stale data.slow TXT comes from cache (stale-answer-client-timeout 1.8) ($n)"
   1804 ret=0
   1805 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1806 grep "EDE: 3 (Stale Answer): (client timeout)" dig.out.test$n >/dev/null || ret=1
   1807 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1808 grep "data\.slow\..*3.*IN.*TXT.*A slow text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   1809 if [ $ret != 0 ]; then echo_i "failed"; fi
   1810 status=$((status + ret))
   1811 
   1812 # Now query for RRset not in cache. The first query should time out, but once
   1813 # we enable the authoritative server, the second query should be able to get a
   1814 # response.
   1815 
   1816 nextpart ns3/named.run >/dev/null
   1817 
   1818 echo_i "sending queries for tests $((n + 2))-$((n + 4))..."
   1819 # first dig runs in background for 10 seconds, second in background for 3
   1820 # seconds and the last for 3 seconds in the foreground.
   1821 # the second RRSIG lookup triggers the issue in [GL #3622]
   1822 $DIG -p ${PORT} +tries=1 +timeout=10 @10.53.0.3 longttl.example TXT >dig.out.test$((n + 3)) &
   1823 $DIG -p ${PORT} +tries=1 +timeout=3 @10.53.0.3 longttl.example RRSIG >dig.out.test$((n + 4)) &
   1824 $DIG -p ${PORT} +tries=1 +timeout=3 @10.53.0.3 longttl.example TXT >dig.out.test$((n + 2)) || true
   1825 
   1826 # Enable the authoritative name server after stale-answer-client-timeout.
   1827 n=$((n + 1))
   1828 echo_i "enable responses from authoritative server ($n)"
   1829 ret=0
   1830 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
   1831 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1832 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   1833 if [ $ret != 0 ]; then echo_i "failed"; fi
   1834 status=$((status + ret))
   1835 
   1836 n=$((n + 1))
   1837 echo_i "check not in cache longttl.example TXT times out (stale-answer-client-timeout 1.8) ($n)"
   1838 ret=0
   1839 wait_for_log 4 "longttl.example client timeout, stale answer unavailable" ns3/named.run || ret=1
   1840 grep "timed out" dig.out.test$n >/dev/null || ret=1
   1841 grep ";; no servers could be reached" dig.out.test$n >/dev/null || ret=1
   1842 if [ $ret != 0 ]; then echo_i "failed"; fi
   1843 status=$((status + ret))
   1844 
   1845 wait
   1846 
   1847 n=$((n + 1))
   1848 echo_i "check not in cache longttl.example TXT comes from authoritative (stale-answer-client-timeout 1.8) ($n)"
   1849 ret=0
   1850 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1851 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1852 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1853 if [ $ret != 0 ]; then echo_i "failed"; fi
   1854 status=$((status + ret))
   1855 
   1856 n=$((n + 1))
   1857 echo_i "check not in cache longttl.example RRSIG times out (stale-answer-client-timeout 1.8) ($n)"
   1858 ret=0
   1859 grep "timed out" dig.out.test$n >/dev/null || ret=1
   1860 grep ";; no servers could be reached" dig.out.test$n >/dev/null || ret=1
   1861 if [ $ret != 0 ]; then echo_i "failed"; fi
   1862 status=$((status + ret))
   1863 
   1864 # CVE-2022-3924, GL #3619
   1865 n=$((n + 1))
   1866 echo_i "check that named survives reaching recursive-clients quota (stale-answer-client-timeout 1.8) ($n)"
   1867 ret=0
   1868 num=0
   1869 # Make sure to exceed the configured value of 'recursive-clients 10;' by running
   1870 # 20 parallel queries with simulated network latency.
   1871 while [ $num -lt 20 ]; do
   1872   $DIG +tries=1 -p ${PORT} @10.53.0.3 "latency${num}.data.example" TXT >/dev/null 2>&1 &
   1873   num=$((num + 1))
   1874 done
   1875 check_server_responds() {
   1876   $DIG -p ${PORT} @10.53.0.3 version.bind txt ch >dig.out.test$n || return 1
   1877   grep "status: NOERROR" dig.out.test$n >/dev/null || return 1
   1878 }
   1879 retry_quiet 5 check_server_responds || ret=1
   1880 if [ $ret != 0 ]; then echo_i "failed"; fi
   1881 status=$((status + ret))
   1882 
   1883 #############################################
   1884 # Test for stale-answer-client-timeout off. #
   1885 #############################################
   1886 echo_i "test stale-answer-client-timeout (off)"
   1887 
   1888 n=$((n + 1))
   1889 echo_i "updating ns3/named.conf ($n)"
   1890 ret=0
   1891 copy_setports ns3/named3.conf.in ns3/named.conf
   1892 if [ $ret != 0 ]; then echo_i "failed"; fi
   1893 status=$((status + ret))
   1894 
   1895 n=$((n + 1))
   1896 echo_i "running 'rndc reload' ($n)"
   1897 ret=0
   1898 rndc_reload ns3 10.53.0.3
   1899 if [ $ret != 0 ]; then echo_i "failed"; fi
   1900 status=$((status + ret))
   1901 
   1902 # Send a query, auth server is disabled, we will enable it after a while in
   1903 # order to receive an answer before resolver-query-timeout expires. Since
   1904 # stale-answer-client-timeout is disabled we must receive an answer from
   1905 # authoritative server.
   1906 echo_i "sending query for test $((n + 2))"
   1907 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$((n + 2)) &
   1908 sleep 3
   1909 
   1910 n=$((n + 1))
   1911 echo_i "enable responses from authoritative server ($n)"
   1912 ret=0
   1913 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
   1914 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1915 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   1916 if [ $ret != 0 ]; then echo_i "failed"; fi
   1917 status=$((status + ret))
   1918 
   1919 # Wait until dig is done.
   1920 wait
   1921 
   1922 n=$((n + 1))
   1923 echo_i "check data.example TXT comes from authoritative server (stale-answer-client-timeout off) ($n)"
   1924 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1925 grep "EDE" dig.out.test$n >/dev/null && ret=1
   1926 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1927 grep "data\.example\..*[12].*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   1928 if [ $ret != 0 ]; then echo_i "failed"; fi
   1929 status=$((status + ret))
   1930 
   1931 ##############################################################
   1932 # Test for stale-answer-client-timeout off and CNAME record. #
   1933 ##############################################################
   1934 echo_i "test stale-answer-client-timeout (0) and CNAME record"
   1935 
   1936 n=$((n + 1))
   1937 echo_i "prime cache shortttl.cname.example (stale-answer-client-timeout off) ($n)"
   1938 ret=0
   1939 $DIG -p ${PORT} @10.53.0.3 shortttl.cname.example A >dig.out.test$n || ret=1
   1940 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1941 grep "ANSWER: 2," dig.out.test$n >/dev/null || ret=1
   1942 grep "shortttl\.cname\.example\..*1.*IN.*CNAME.*longttl\.target\.example\." dig.out.test$n >/dev/null || ret=1
   1943 grep "longttl\.target\.example\..*600.*IN.*A.*10\.53\.0\.2" dig.out.test$n >/dev/null || ret=1
   1944 if [ $ret != 0 ]; then echo_i "failed"; fi
   1945 status=$((status + ret))
   1946 
   1947 # Allow RRset to become stale.
   1948 sleep 1
   1949 
   1950 n=$((n + 1))
   1951 echo_i "disable responses from authoritative server ($n)"
   1952 ret=0
   1953 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
   1954 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1955 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
   1956 if [ $ret != 0 ]; then echo_i "failed"; fi
   1957 status=$((status + ret))
   1958 
   1959 n=$((n + 1))
   1960 ret=0
   1961 echo_i "check stale shortttl.cname.example comes from cache (stale-answer-client-timeout off) ($n)"
   1962 nextpart ns3/named.run >/dev/null
   1963 $DIG -p ${PORT} @10.53.0.3 shortttl.cname.example A >dig.out.test$n || ret=1
   1964 wait_for_log 5 "shortttl.cname.example resolver failure, stale answer used" ns3/named.run || ret=1
   1965 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   1966 grep "EDE: 3 (Stale Answer): (resolver failure)" dig.out.test$n >/dev/null || ret=1
   1967 grep "ANSWER: 2," dig.out.test$n >/dev/null || ret=1
   1968 grep "shortttl\.cname\.example\..*3.*IN.*CNAME.*longttl\.target\.example\." dig.out.test$n >/dev/null || ret=1
   1969 # We can't reliably test the TTL of the longttl.target.example A record.
   1970 grep "longttl\.target\.example\..*IN.*A.*10\.53\.0\.2" dig.out.test$n >/dev/null || ret=1
   1971 if [ $ret != 0 ]; then echo_i "failed"; fi
   1972 status=$((status + ret))
   1973 
   1974 n=$((n + 1))
   1975 echo_i "enable responses from authoritative server ($n)"
   1976 ret=0
   1977 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
   1978 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   1979 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   1980 if [ $ret != 0 ]; then echo_i "failed"; fi
   1981 status=$((status + ret))
   1982 
   1983 n=$((n + 1))
   1984 echo_i "check server is alive or restart ($n)"
   1985 ret=0
   1986 $RNDCCMD 10.53.0.3 status >rndc.out.test$n 2>&1 || ret=1
   1987 if [ $ret != 0 ]; then
   1988   echo_i "failed"
   1989   echo_i "restart ns3"
   1990   start_server --noclean --restart --port ${PORT} serve-stale ns3
   1991 fi
   1992 status=$((status + ret))
   1993 
   1994 n=$((n + 1))
   1995 echo_i "check server is alive or restart ($n)"
   1996 ret=0
   1997 $RNDCCMD 10.53.0.3 status >rndc.out.test$n 2>&1 || ret=1
   1998 if [ $ret != 0 ]; then
   1999   echo_i "failed"
   2000   echo_i "restart ns3"
   2001   start_server --noclean --restart --port ${PORT} serve-stale ns3
   2002 fi
   2003 status=$((status + ret))
   2004 
   2005 #############################################
   2006 # Test for stale-answer-client-timeout 0.   #
   2007 #############################################
   2008 echo_i "test stale-answer-client-timeout (0)"
   2009 
   2010 n=$((n + 1))
   2011 echo_i "updating ns3/named.conf ($n)"
   2012 ret=0
   2013 copy_setports ns3/named4.conf.in ns3/named.conf
   2014 if [ $ret != 0 ]; then echo_i "failed"; fi
   2015 status=$((status + ret))
   2016 
   2017 echo_i "restart ns3"
   2018 stop_server --use-rndc --port ${CONTROLPORT} ns3
   2019 start_server --noclean --restart --port ${PORT} ns3
   2020 
   2021 n=$((n + 1))
   2022 echo_i "prime cache data.example TXT (stale-answer-client-timeout 0)"
   2023 ret=0
   2024 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2025 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2026 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2027 if [ $ret != 0 ]; then echo_i "failed"; fi
   2028 status=$((status + ret))
   2029 
   2030 n=$((n + 1))
   2031 echo_i "prime cache nodata.example TXT (stale-answer-client-timeout 0)"
   2032 ret=0
   2033 $DIG -p ${PORT} @10.53.0.3 nodata.example TXT >dig.out.test$n || ret=1
   2034 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2035 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   2036 if [ $ret != 0 ]; then echo_i "failed"; fi
   2037 status=$((status + ret))
   2038 
   2039 n=$((n + 1))
   2040 echo_i "disable responses from authoritative server ($n)"
   2041 ret=0
   2042 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
   2043 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2044 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
   2045 if [ $ret != 0 ]; then echo_i "failed"; fi
   2046 status=$((status + ret))
   2047 
   2048 # Allow RRset to become stale.
   2049 sleep 2
   2050 
   2051 n=$((n + 1))
   2052 ret=0
   2053 echo_i "check stale nodata.example TXT comes from cache (stale-answer-client-timeout 0) ($n)"
   2054 nextpart ns3/named.run >/dev/null
   2055 $DIG -p ${PORT} @10.53.0.3 nodata.example TXT >dig.out.test$n || ret=1
   2056 wait_for_log 5 "nodata.example stale answer used, an attempt to refresh the RRset" ns3/named.run || ret=1
   2057 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2058 grep "EDE: 3 (Stale Answer): (stale data prioritized over lookup)" dig.out.test$n >/dev/null || ret=1
   2059 grep "ANSWER: 0," dig.out.test$n >/dev/null || ret=1
   2060 grep "example\..*3.*IN.*SOA" dig.out.test$n >/dev/null || ret=1
   2061 if [ $ret != 0 ]; then echo_i "failed"; fi
   2062 status=$((status + ret))
   2063 
   2064 n=$((n + 1))
   2065 ret=0
   2066 echo_i "check stale data.example TXT comes from cache (stale-answer-client-timeout 0) ($n)"
   2067 nextpart ns3/named.run >/dev/null
   2068 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2069 wait_for_log 5 "data.example stale answer used, an attempt to refresh the RRset" ns3/named.run || ret=1
   2070 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2071 grep "EDE: 3 (Stale Answer): (stale data prioritized over lookup)" dig.out.test$n >/dev/null || ret=1
   2072 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2073 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   2074 if [ $ret != 0 ]; then echo_i "failed"; fi
   2075 status=$((status + ret))
   2076 
   2077 n=$((n + 1))
   2078 echo_i "enable responses from authoritative server ($n)"
   2079 ret=0
   2080 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
   2081 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2082 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   2083 if [ $ret != 0 ]; then echo_i "failed"; fi
   2084 status=$((status + ret))
   2085 
   2086 wait_for_rrset_refresh() {
   2087   $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || return 1
   2088   grep "status: NOERROR" dig.out.test$n >/dev/null || return 1
   2089   grep "EDE" dig.out.test$n >/dev/null && return 1
   2090   grep "ANSWER: 1," dig.out.test$n >/dev/null || return 1
   2091   grep "data\.example\..*[12].*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || return 1
   2092 }
   2093 
   2094 # This test ensures that after we get stale data due to
   2095 # stale-answer-client-timeout 0, enabling the authoritative server will allow
   2096 # the RRset to be updated.
   2097 n=$((n + 1))
   2098 ret=0
   2099 echo_i "check stale data.example TXT was refreshed (stale-answer-client-timeout 0) ($n)"
   2100 retry_quiet 10 wait_for_rrset_refresh || ret=1
   2101 if [ $ret != 0 ]; then echo_i "failed"; fi
   2102 status=$((status + ret))
   2103 
   2104 wait_for_nodata_refresh() {
   2105   $DIG -p ${PORT} @10.53.0.3 nodata.example TXT >dig.out.test$n || return 1
   2106   grep "status: NOERROR" dig.out.test$n >/dev/null || return 1
   2107   grep "ANSWER: 0," dig.out.test$n >/dev/null || return 1
   2108   grep "example\..*[12].*IN.*SOA" dig.out.test$n >/dev/null || return 1
   2109   return 0
   2110 }
   2111 
   2112 n=$((n + 1))
   2113 ret=0
   2114 echo_i "check stale nodata.example TXT was refreshed (stale-answer-client-timeout 0) ($n)"
   2115 retry_quiet 10 wait_for_nodata_refresh || ret=1
   2116 if [ $ret != 0 ]; then echo_i "failed"; fi
   2117 status=$((status + ret))
   2118 
   2119 ####################################################################
   2120 # Test for stale-answer-client-timeout 0 and recursive-clients 10. #
   2121 # CVE-2023-2911, GL #4089                                          #
   2122 # ##################################################################
   2123 echo_i "test stale-answer-client-timeout (0) and recursive-clients 10"
   2124 
   2125 n=$((n + 1))
   2126 echo_i "prime cache data.slow TXT (stale-answer-client-timeout 0) ($n)"
   2127 ret=0
   2128 $DIG -p ${PORT} @10.53.0.3 data.slow TXT >dig.out.test$n || ret=1
   2129 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2130 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2131 if [ $ret != 0 ]; then echo_i "failed"; fi
   2132 status=$((status + ret))
   2133 
   2134 # Run the following check twice. Sometimes a priming query interrupts the first
   2135 # attempt to exceed the quota.
   2136 attempt=0
   2137 while [ $ret -eq 0 ] && [ $attempt -lt 2 ]; do
   2138   n=$((n + 1))
   2139   echo_i "slow down response from authoritative server ($n)"
   2140   ret=0
   2141   $DIG -p ${PORT} @10.53.0.2 slowdown TXT >dig.out.test$n || ret=1
   2142   grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2143   grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   2144   if [ $ret != 0 ]; then echo_i "failed"; fi
   2145   status=$((status + ret))
   2146 
   2147   # Let the data.slow TTL expire
   2148   sleep 2
   2149 
   2150   n=$((n + 1))
   2151   echo_i "check that named survives reaching recursive-clients quota (stale-answer-client-timeout 0) ($n)"
   2152   ret=0
   2153   num=0
   2154   # Attempt to exceed the configured value of 'recursive-clients 10;' by running
   2155   # 20 parallel queries for the stale domain which has slow auth.
   2156   while [ $num -lt 20 ]; do
   2157     $DIG +tries=1 +timeout=10 -p ${PORT} @10.53.0.3 data.slow TXT >/dev/null 2>&1 &
   2158     num=$((num + 1))
   2159   done
   2160   # Let the dig processes finish.
   2161   wait
   2162   retry_quiet 5 check_server_responds || ret=1
   2163   if [ $ret != 0 ]; then echo_i "failed"; fi
   2164   status=$((status + ret))
   2165 
   2166   attempt=$((attempt + 1))
   2167 done
   2168 
   2169 # Restart ns3 to avoid the exceeded recursive-clients limit from previous check
   2170 # to interfere with subsequent checks.
   2171 echo_i "restart ns3"
   2172 stop_server --use-rndc --port ${CONTROLPORT} ns3
   2173 start_server --noclean --restart --port ${PORT} ns3
   2174 
   2175 ############################################################
   2176 # Test for stale-answer-client-timeout 0 and CNAME record. #
   2177 ############################################################
   2178 echo_i "test stale-answer-client-timeout (0) and CNAME record"
   2179 
   2180 n=$((n + 1))
   2181 echo_i "prime cache cname1.stale.test A (stale-answer-client-timeout 0) ($n)"
   2182 ret=0
   2183 $DIG -p ${PORT} @10.53.0.3 cname1.stale.test A >dig.out.test$n || ret=1
   2184 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2185 grep "ANSWER: 2," dig.out.test$n >/dev/null || ret=1
   2186 grep "cname1\.stale\.test\..*1.*IN.*CNAME.*a1\.stale\.test\." dig.out.test$n >/dev/null || ret=1
   2187 grep "a1\.stale\.test\..*1.*IN.*A.*192\.0\.2\.1" dig.out.test$n >/dev/null || ret=1
   2188 if [ $ret != 0 ]; then echo_i "failed"; fi
   2189 status=$((status + ret))
   2190 
   2191 # Allow RRset to become stale.
   2192 sleep 1
   2193 
   2194 n=$((n + 1))
   2195 ret=0
   2196 echo_i "check stale cname1.stale.test A comes from cache (stale-answer-client-timeout 0) ($n)"
   2197 nextpart ns3/named.run >/dev/null
   2198 $DIG -p ${PORT} @10.53.0.3 cname1.stale.test A >dig.out.test$n || ret=1
   2199 wait_for_log 5 "cname1.stale.test stale answer used, an attempt to refresh the RRset" ns3/named.run || ret=1
   2200 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2201 grep "EDE: 3 (Stale Answer): (stale data prioritized over lookup)" dig.out.test$n >/dev/null || ret=1
   2202 grep "ANSWER: 2," dig.out.test$n >/dev/null || ret=1
   2203 grep "cname1\.stale\.test\..*3.*IN.*CNAME.*a1\.stale\.test\." dig.out.test$n >/dev/null || ret=1
   2204 grep "a1\.stale\.test\..*3.*IN.*A.*192\.0\.2\.1" dig.out.test$n >/dev/null || ret=1
   2205 if [ $ret != 0 ]; then echo_i "failed"; fi
   2206 status=$((status + ret))
   2207 
   2208 n=$((n + 1))
   2209 echo_i "check server is alive or restart ($n)"
   2210 ret=0
   2211 $RNDCCMD 10.53.0.3 status >rndc.out.test$n 2>&1 || ret=1
   2212 if [ $ret != 0 ]; then
   2213   echo_i "failed"
   2214   echo_i "restart ns3"
   2215   start_server --noclean --restart --port ${PORT} ns3
   2216 fi
   2217 status=$((status + ret))
   2218 
   2219 n=$((n + 1))
   2220 echo_i "prime cache cname2.stale.test A (stale-answer-client-timeout 0) ($n)"
   2221 ret=0
   2222 $DIG -p ${PORT} @10.53.0.3 cname2.stale.test A >dig.out.test$n || ret=1
   2223 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2224 grep "ANSWER: 2," dig.out.test$n >/dev/null || ret=1
   2225 grep "cname2\.stale\.test\..*1.*IN.*CNAME.*a2\.stale\.test\." dig.out.test$n >/dev/null || ret=1
   2226 grep "a2\.stale\.test\..*300.*IN.*A.*192\.0\.2\.2" dig.out.test$n >/dev/null || ret=1
   2227 if [ $ret != 0 ]; then echo_i "failed"; fi
   2228 status=$((status + ret))
   2229 
   2230 # Allow CNAME record in the RRSET to become stale.
   2231 sleep 1
   2232 
   2233 n=$((n + 1))
   2234 ret=0
   2235 echo_i "check stale cname2.stale.test A comes from cache (stale-answer-client-timeout 0) ($n)"
   2236 nextpart ns3/named.run >/dev/null
   2237 $DIG -p ${PORT} @10.53.0.3 cname2.stale.test A >dig.out.test$n || ret=1
   2238 wait_for_log 5 "cname2.stale.test stale answer used, an attempt to refresh the RRset" ns3/named.run || ret=1
   2239 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2240 grep "EDE: 3 (Stale Answer): (stale data prioritized over lookup)" dig.out.test$n >/dev/null || ret=1
   2241 grep "ANSWER: 2," dig.out.test$n >/dev/null || ret=1
   2242 grep "cname2\.stale\.test\..*3.*IN.*CNAME.*a2\.stale\.test\." dig.out.test$n >/dev/null || ret=1
   2243 # We can't reliably test the TTL of the a2.stale.test A record.
   2244 grep "a2\.stale\.test\..*IN.*A.*192\.0\.2\.2" dig.out.test$n >/dev/null || ret=1
   2245 if [ $ret != 0 ]; then echo_i "failed"; fi
   2246 status=$((status + ret))
   2247 
   2248 n=$((n + 1))
   2249 echo_i "check server is alive or restart ($n)"
   2250 ret=0
   2251 $RNDCCMD 10.53.0.3 status >rndc.out.test$n 2>&1 || ret=1
   2252 if [ $ret != 0 ]; then
   2253   echo_i "failed"
   2254   echo_i "restart ns3"
   2255   start_server --noclean --restart --port ${PORT} ns3
   2256 fi
   2257 status=$((status + ret))
   2258 
   2259 ####################################################################
   2260 # Test for stale-answer-client-timeout 0 and stale-refresh-time 4. #
   2261 ####################################################################
   2262 echo_i "test stale-answer-client-timeout (0) and stale-refresh-time (4)"
   2263 
   2264 n=$((n + 1))
   2265 echo_i "updating ns3/named.conf ($n)"
   2266 ret=0
   2267 copy_setports ns3/named5.conf.in ns3/named.conf
   2268 if [ $ret != 0 ]; then echo_i "failed"; fi
   2269 status=$((status + ret))
   2270 
   2271 n=$((n + 1))
   2272 echo_i "running 'rndc reload' ($n)"
   2273 ret=0
   2274 rndc_reload ns3 10.53.0.3
   2275 if [ $ret != 0 ]; then echo_i "failed"; fi
   2276 status=$((status + ret))
   2277 
   2278 n=$((n + 1))
   2279 echo_i "flush cache, enable responses from authoritative server ($n)"
   2280 ret=0
   2281 $RNDCCMD 10.53.0.3 flushtree example >rndc.out.test$n.1 2>&1 || ret=1
   2282 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
   2283 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2284 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   2285 if [ $ret != 0 ]; then echo_i "failed"; fi
   2286 status=$((status + ret))
   2287 
   2288 n=$((n + 1))
   2289 echo_i "prime cache data.example TXT (stale-answer-client-timeout 0, stale-refresh-time 4) ($n)"
   2290 ret=0
   2291 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2292 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2293 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2294 grep "data\.example\..*2.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   2295 if [ $ret != 0 ]; then echo_i "failed"; fi
   2296 status=$((status + ret))
   2297 
   2298 # Allow RRset to become stale.
   2299 sleep 2
   2300 
   2301 n=$((n + 1))
   2302 echo_i "disable responses from authoritative server ($n)"
   2303 ret=0
   2304 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
   2305 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2306 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
   2307 if [ $ret != 0 ]; then echo_i "failed"; fi
   2308 status=$((status + ret))
   2309 
   2310 n=$((n + 1))
   2311 ret=0
   2312 echo_i "check stale data.example TXT comes from cache (stale-answer-client-timeout 0 stale-refresh-time 4) ($n)"
   2313 nextpart ns3/named.run >/dev/null
   2314 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2315 wait_for_log 5 "data.example stale answer used, an attempt to refresh the RRset" ns3/named.run || ret=1
   2316 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2317 grep "EDE: 3 (Stale Answer): (stale data prioritized over lookup)" dig.out.test$n >/dev/null || ret=1
   2318 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2319 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   2320 if [ $ret != 0 ]; then echo_i "failed"; fi
   2321 status=$((status + ret))
   2322 
   2323 n=$((n + 1))
   2324 echo_i "enable responses from authoritative server ($n)"
   2325 ret=0
   2326 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
   2327 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2328 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   2329 if [ $ret != 0 ]; then echo_i "failed"; fi
   2330 status=$((status + ret))
   2331 
   2332 # This test ensures that after we get stale data due to
   2333 # stale-answer-client-timeout 0, enabling the authoritative server will allow
   2334 # the RRset to be updated.
   2335 n=$((n + 1))
   2336 ret=0
   2337 echo_i "check stale data.example TXT was refreshed (stale-answer-client-timeout 0 stale-refresh-time 4) ($n)"
   2338 retry_quiet 10 wait_for_rrset_refresh || ret=1
   2339 if [ $ret != 0 ]; then echo_i "failed"; fi
   2340 status=$((status + ret))
   2341 
   2342 # Allow RRset to become stale.
   2343 sleep 2
   2344 
   2345 n=$((n + 1))
   2346 echo_i "disable responses from authoritative server ($n)"
   2347 ret=0
   2348 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
   2349 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2350 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
   2351 if [ $ret != 0 ]; then echo_i "failed"; fi
   2352 status=$((status + ret))
   2353 
   2354 n=$((n + 1))
   2355 ret=0
   2356 echo_i "check stale data.example TXT comes from cache (stale-answer-client-timeout 0 stale-refresh-time 4) ($n)"
   2357 nextpart ns3/named.run >/dev/null
   2358 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2359 wait_for_log 5 "data.example stale answer used, an attempt to refresh the RRset" ns3/named.run || ret=1
   2360 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2361 grep "EDE: 3 (Stale Answer): (stale data prioritized over lookup)" dig.out.test$n >/dev/null || ret=1
   2362 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2363 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   2364 if [ $ret != 0 ]; then echo_i "failed"; fi
   2365 status=$((status + ret))
   2366 
   2367 # Allow stale-refresh-time to be activated.
   2368 n=$((n + 1))
   2369 ret=0
   2370 echo_i "wait until resolver query times out, activating stale-refresh-time"
   2371 wait_for_log 15 "data.example resolver failure, stale answer used" ns3/named.run || ret=1
   2372 if [ $ret != 0 ]; then echo_i "failed"; fi
   2373 status=$((status + ret))
   2374 
   2375 n=$((n + 1))
   2376 ret=0
   2377 echo_i "check stale data.example TXT comes from cache within stale-refresh-time (stale-answer-client-timeout 0 stale-refresh-time 4) ($n)"
   2378 nextpart ns3/named.run >/dev/null
   2379 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2380 wait_for_log 5 "data.example query within stale refresh time" ns3/named.run || ret=1
   2381 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2382 grep "EDE: 3 (Stale Answer): (query within stale refresh time window)" dig.out.test$n >/dev/null || ret=1
   2383 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2384 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   2385 if [ $ret != 0 ]; then echo_i "failed"; fi
   2386 status=$((status + ret))
   2387 
   2388 n=$((n + 1))
   2389 echo_i "enable responses from authoritative server ($n)"
   2390 ret=0
   2391 $DIG -p ${PORT} @10.53.0.2 txt enable >dig.out.test$n || ret=1
   2392 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2393 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   2394 if [ $ret != 0 ]; then echo_i "failed"; fi
   2395 status=$((status + ret))
   2396 
   2397 # We give BIND some time to ensure that after we enable authoritative server,
   2398 # this RRset is still not refreshed because it was hit during
   2399 # stale-refresh-time window.
   2400 sleep 1
   2401 
   2402 n=$((n + 1))
   2403 ret=0
   2404 echo_i "check stale data.example TXT was not refreshed (stale-answer-client-timeout 0 stale-refresh-time 4) ($n)"
   2405 nextpart ns3/named.run >/dev/null
   2406 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2407 wait_for_log 5 "data.example query within stale refresh time" ns3/named.run || ret=1
   2408 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2409 grep "EDE: 3 (Stale Answer): (query within stale refresh time window)" dig.out.test$n >/dev/null || ret=1
   2410 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2411 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   2412 if [ $ret != 0 ]; then echo_i "failed"; fi
   2413 status=$((status + ret))
   2414 
   2415 # After the refresh-time-window, the RRset will be refreshed.
   2416 sleep 4
   2417 
   2418 n=$((n + 1))
   2419 ret=0
   2420 echo_i "check stale data.example TXT comes from cache (stale-answer-client-timeout 0 stale-refresh-time 4) ($n)"
   2421 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2422 wait_for_log 5 "data.example stale answer used, an attempt to refresh the RRset" ns3/named.run || ret=1
   2423 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2424 grep "EDE: 3 (Stale Answer): (stale data prioritized over lookup)" dig.out.test$n >/dev/null || ret=1
   2425 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2426 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   2427 if [ $ret != 0 ]; then echo_i "failed"; fi
   2428 status=$((status + ret))
   2429 
   2430 n=$((n + 1))
   2431 ret=0
   2432 echo_i "check stale data.example TXT was refreshed (stale-answer-client-timeout 0 stale-refresh-time 4) ($n)"
   2433 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2434 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2435 grep "EDE" dig.out.test$n >/dev/null && ret=1
   2436 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2437 grep "data\.example\..*[12].*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   2438 if [ $ret != 0 ]; then echo_i "failed"; fi
   2439 status=$((status + ret))
   2440 
   2441 ####################################################################
   2442 # Test serve-stale's interaction with fetch limits (cache only) #
   2443 #################################################################
   2444 echo_i "test serve-stale's interaction with fetch-limits (cache only)"
   2445 
   2446 # We update the named configuration to enable fetch-limits. The fetch-limits
   2447 # are set to 1, which is ridiciously low, but that is because for this test we
   2448 # want to reach the fetch-limits.
   2449 n=$((n + 1))
   2450 echo_i "updating ns3/named.conf ($n)"
   2451 ret=0
   2452 copy_setports ns3/named6.conf.in ns3/named.conf
   2453 if [ $ret != 0 ]; then echo_i "failed"; fi
   2454 status=$((status + ret))
   2455 
   2456 n=$((n + 1))
   2457 echo_i "running 'rndc reload' ($n)"
   2458 ret=0
   2459 rndc_reload ns3 10.53.0.3
   2460 if [ $ret != 0 ]; then echo_i "failed"; fi
   2461 status=$((status + ret))
   2462 
   2463 # Disable responses from authoritative server. If we can't resolve the example
   2464 # zone, fetch limits will be reached.
   2465 n=$((n + 1))
   2466 echo_i "disable responses from authoritative server ($n)"
   2467 ret=0
   2468 $DIG -p ${PORT} @10.53.0.2 txt disable >dig.out.test$n || ret=1
   2469 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2470 grep "TXT.\"0\"" dig.out.test$n >/dev/null || ret=1
   2471 if [ $ret != 0 ]; then echo_i "failed"; fi
   2472 status=$((status + ret))
   2473 
   2474 # Allow RRset to become stale.
   2475 sleep 2
   2476 
   2477 # Turn on serve-stale.
   2478 n=$((n + 1))
   2479 echo_i "running 'rndc serve-stale on' ($n)"
   2480 ret=0
   2481 $RNDCCMD 10.53.0.3 serve-stale on || ret=1
   2482 if [ $ret != 0 ]; then echo_i "failed"; fi
   2483 status=$((status + ret))
   2484 
   2485 n=$((n + 1))
   2486 echo_i "check 'rndc serve-stale status' ($n)"
   2487 ret=0
   2488 $RNDCCMD 10.53.0.3 serve-stale status >rndc.out.test$n 2>&1 || ret=1
   2489 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=3 max-stale-ttl=3600 stale-refresh-time=4)' rndc.out.test$n >/dev/null || ret=1
   2490 if [ $ret != 0 ]; then echo_i "failed"; fi
   2491 status=$((status + ret))
   2492 
   2493 # Hit the fetch-limits. We burst the name server with a small batch of queries.
   2494 # Only 2 queries are required to hit the fetch-limits. The first query will
   2495 # start to resolve, the second one hit the fetch-limits.
   2496 burst() {
   2497   num=${1}
   2498   rm -f burst.input.$$
   2499   while [ $num -gt 0 ]; do
   2500     num=$((num - 1))
   2501     echo "fetch${num}.example A" >>burst.input.$$
   2502   done
   2503   $PERL ../ditch.pl -p ${PORT} -s 10.53.0.3 -b ${EXTRAPORT8} burst.input.$$
   2504   rm -f burst.input.$$
   2505 }
   2506 
   2507 wait_for_fetchlimits() {
   2508   burst 2
   2509   # We expect a query for nx.example to fail because fetch-limits for
   2510   # the domain 'example.' (and everything below) has been reached.
   2511   $DIG -p ${PORT} +tries=1 +timeout=1 @10.53.0.3 nx.example >dig.out.test$n || return 1
   2512   grep "status: SERVFAIL" dig.out.test$n >/dev/null || return 1
   2513 }
   2514 
   2515 n=$((n + 1))
   2516 echo_i "hit fetch limits ($n)"
   2517 ret=0
   2518 retry_quiet 10 wait_for_fetchlimits || ret=1
   2519 if [ $ret != 0 ]; then echo_i "failed"; fi
   2520 status=$((status + ret))
   2521 
   2522 # Expect stale data now (because fetch-limits for the domain 'example.' (and
   2523 # everything below) has been reached. But we have a stale RRset for
   2524 # 'data.example/TXT' that can be used.
   2525 n=$((n + 1))
   2526 ret=0
   2527 echo_i "check stale data.example TXT comes from cache (fetch-limits) ($n)"
   2528 nextpart ns3/named.run >/dev/null
   2529 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2530 wait_for_log 5 "data.example resolver failure, stale answer used" ns3/named.run || ret=1
   2531 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2532 grep "EDE: 3 (Stale Answer): (resolver failure" dig.out.test$n >/dev/null || ret=1
   2533 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2534 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   2535 if [ $ret != 0 ]; then echo_i "failed"; fi
   2536 status=$((status + ret))
   2537 
   2538 # The previous query should not have started the stale-refresh-time window.
   2539 n=$((n + 1))
   2540 ret=0
   2541 echo_i "check stale data.example TXT comes from cache again (fetch-limits) ($n)"
   2542 nextpart ns3/named.run >/dev/null
   2543 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$n || ret=1
   2544 wait_for_log 5 "data.example resolver failure, stale answer used" ns3/named.run || ret=1
   2545 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2546 grep "EDE: 3 (Stale Answer): (resolver failure" dig.out.test$n >/dev/null || ret=1
   2547 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2548 grep "data\.example\..*3.*IN.*TXT.*A text record with a 2 second ttl" dig.out.test$n >/dev/null || ret=1
   2549 if [ $ret != 0 ]; then echo_i "failed"; fi
   2550 status=$((status + ret))
   2551 
   2552 ########################################################################
   2553 # Test serve-stale's interaction with fetch limits (dual-mode) #
   2554 ########################################################################
   2555 echo_i "test serve-stale's interaction with fetch limits (dual-mode)"
   2556 
   2557 # Update named configuration so that ns3 becomes a recursive resolver which is
   2558 # also a secondary server for the root zone.
   2559 n=$((n + 1))
   2560 echo_i "updating ns3/named.conf ($n)"
   2561 ret=0
   2562 copy_setports ns3/named7.conf.in ns3/named.conf
   2563 if [ $ret != 0 ]; then echo_i "failed"; fi
   2564 status=$((status + ret))
   2565 
   2566 n=$((n + 1))
   2567 echo_i "running 'rndc reload' ($n)"
   2568 ret=0
   2569 rndc_reload ns3 10.53.0.3
   2570 if [ $ret != 0 ]; then echo_i "failed"; fi
   2571 status=$((status + ret))
   2572 
   2573 n=$((n + 1))
   2574 echo_i "check 'rndc serve-stale status' ($n)"
   2575 ret=0
   2576 $RNDCCMD 10.53.0.3 serve-stale status >rndc.out.test$n 2>&1 || ret=1
   2577 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=3 max-stale-ttl=3600 stale-refresh-time=4)' rndc.out.test$n >/dev/null || ret=1
   2578 if [ $ret != 0 ]; then echo_i "failed"; fi
   2579 status=$((status + ret))
   2580 
   2581 # Flush the cache to ensure the example/NS RRset cached during previous tests
   2582 # does not override the authoritative delegation found in the root zone.
   2583 n=$((n + 1))
   2584 echo_i "flush cache ($n)"
   2585 ret=0
   2586 $RNDCCMD 10.53.0.3 flush >rndc.out.test$n 2>&1 || ret=1
   2587 if [ $ret != 0 ]; then echo_i "failed"; fi
   2588 status=$((status + ret))
   2589 
   2590 # Test that after flush, serve-stale configuration is not reset.
   2591 n=$((n + 1))
   2592 echo_i "check serve-stale configuration is not reset after flush ($n)"
   2593 ret=0
   2594 $RNDCCMD 10.53.0.3 serve-stale status >rndc.out.test$n 2>&1 || ret=1
   2595 grep '_default: stale cache enabled; stale answers enabled (stale-answer-ttl=3 max-stale-ttl=3600 stale-refresh-time=4)' rndc.out.test$n >/dev/null || ret=1
   2596 if [ $ret != 0 ]; then echo_i "failed"; fi
   2597 status=$((status + ret))
   2598 
   2599 # Query name server with low fetch limits. The authoritative server (ans2) is
   2600 # not responding. Sending queries for multiple names in the 'example' zone
   2601 # in parallel causes the fetch limit for that zone (set to 1) to be
   2602 # reached. This should not trigger a crash.
   2603 echo_i "sending queries for tests $((n + 1))-$((n + 4))..."
   2604 $DIG -p ${PORT} @10.53.0.3 data.example TXT >dig.out.test$((n + 1)) &
   2605 $DIG -p ${PORT} @10.53.0.3 othertype.example CAA >dig.out.test$((n + 2)) &
   2606 $DIG -p ${PORT} @10.53.0.3 nodata.example TXT >dig.out.test$((n + 3)) &
   2607 $DIG -p ${PORT} @10.53.0.3 nxdomain.example TXT >dig.out.test$((n + 4)) &
   2608 
   2609 wait
   2610 
   2611 # Expect SERVFAIL for the entries not in cache.
   2612 n=$((n + 1))
   2613 echo_i "check stale data.example TXT (fetch-limits dual-mode) ($n)"
   2614 ret=0
   2615 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   2616 if [ $ret != 0 ]; then echo_i "failed"; fi
   2617 status=$((status + ret))
   2618 
   2619 n=$((n + 1))
   2620 echo_i "check stale othertype.example CAA (fetch-limits dual-mode) ($n)"
   2621 ret=0
   2622 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   2623 if [ $ret != 0 ]; then echo_i "failed"; fi
   2624 status=$((status + ret))
   2625 
   2626 n=$((n + 1))
   2627 echo_i "check stale nodata.example TXT (fetch-limits dual-mode) ($n)"
   2628 ret=0
   2629 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   2630 if [ $ret != 0 ]; then echo_i "failed"; fi
   2631 status=$((status + ret))
   2632 
   2633 n=$((n + 1))
   2634 echo_i "check stale nxdomain.example TXT (fetch-limits dual-mode) ($n)"
   2635 ret=0
   2636 grep "status: SERVFAIL" dig.out.test$n >/dev/null || ret=1
   2637 if [ $ret != 0 ]; then echo_i "failed"; fi
   2638 status=$((status + ret))
   2639 
   2640 n=$((n + 1))
   2641 echo_i "check DNS64 processing of a stale negative answer ($n)"
   2642 ret=0
   2643 # configure ns3 with dns64
   2644 copy_setports ns3/named8.conf.in ns3/named.conf
   2645 rndc_reload ns3 10.53.0.3
   2646 # flush cache, enable ans2 responses, make sure serve-stale is on
   2647 $RNDCCMD 10.53.0.3 flush >rndc.out.test$n.1 2>&1 || ret=1
   2648 $DIG -p ${PORT} @10.53.0.2 txt enable >/dev/null || ret=1
   2649 $RNDCCMD 10.53.0.3 serve-stale on >rndc.out.test$n.2 2>&1 || ret=1
   2650 # prime the cache with an AAAA NXRRSET response
   2651 $DIG -p ${PORT} @10.53.0.3 a-only.example AAAA >dig.out.1.test$n || ret=1
   2652 grep "status: NOERROR" dig.out.1.test$n >/dev/null || ret=1
   2653 grep "2001:aaaa" dig.out.1.test$n >/dev/null || ret=1
   2654 # disable responses from the auth server
   2655 $DIG -p ${PORT} @10.53.0.2 txt disable >/dev/null || ret=1
   2656 # wait two seconds for the previous answer to become stale
   2657 sleep 2
   2658 # resend the query and wait in the background; we should get a stale answer
   2659 $DIG -p ${PORT} @10.53.0.3 a-only.example AAAA >dig.out.2.test$n &
   2660 # re-enable queries after a pause, so the server gets a real answer too
   2661 sleep 2
   2662 $DIG -p ${PORT} @10.53.0.2 txt enable >/dev/null || ret=1
   2663 wait
   2664 grep "status: NOERROR" dig.out.2.test$n >/dev/null || ret=1
   2665 grep "2001:aaaa" dig.out.2.test$n >/dev/null || ret=1
   2666 if [ $ret != 0 ]; then echo_i "failed"; fi
   2667 status=$((status + ret))
   2668 
   2669 n=$((n + 1))
   2670 echo_i "check DNS64 processing of a stale negative answer (short serve-stale-client-timeout) ($n)"
   2671 ret=0
   2672 # configure ns3 with dns64
   2673 copy_setports ns3/named9.conf.in ns3/named.conf
   2674 $RNDCCMD 10.53.0.3 reload >rndc.out.test$n.1 2>&1 || ret=1
   2675 # flush cache, enable ans2 responses, make sure serve-stale is on
   2676 $RNDCCMD 10.53.0.3 flush >rndc.out.test$n.1 2>&1 || ret=1
   2677 $DIG -p ${PORT} @10.53.0.2 txt enable >/dev/null
   2678 $RNDCCMD 10.53.0.3 serve-stale on >rndc.out.test$n.2 2>&1 || ret=1
   2679 #
   2680 $DIG -p ${PORT} @10.53.0.3 a-only-slow.example AAAA >dig.out.test$n || ret=1
   2681 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2682 grep "2001:aaaa" dig.out.test$n >/dev/null || ret=1
   2683 # revert configuration changes introduced by this check
   2684 copy_setports ns3/named8.conf.in ns3/named.conf
   2685 $RNDCCMD 10.53.0.3 reload >rndc.out.test$n.1 2>&1 || ret=1
   2686 if [ $ret != 0 ]; then echo_i "failed"; fi
   2687 status=$((status + ret))
   2688 
   2689 ###########################################################
   2690 # Test serve-stale's interaction with prefetch processing #
   2691 ###########################################################
   2692 echo_i "test serve-stale's interaction with prefetch processing"
   2693 
   2694 # Test case for #2733, ensuring that prefetch queries do not trigger
   2695 # a lookup due to stale-answer-client-timeout.
   2696 #
   2697 # 1. Cache the following records:
   2698 #    cname.example 7 IN CNAME target.example.
   2699 #    target.example 9 IN A <addr>.
   2700 # 2. Let the CNAME RRset expire.
   2701 # 3. Query for 'cname.example/A'.
   2702 #
   2703 # This starts recursion because cname.example/CNAME is expired.
   2704 # The authoritative server is up so likely it will respond before
   2705 #  stale-answer-client-timeout is triggered.
   2706 # The 'target.example/A' RRset is found in cache with a positive value
   2707 #  and is eligble for prefetching.
   2708 # A prefetch is done for 'target.example/A', our ans2 server will
   2709 #  delay the request.
   2710 # The 'prefetch_done()' callback should have the right event type
   2711 #  (DNS_EVENT_FETCHDONE).
   2712 
   2713 # flush cache
   2714 n=$((n + 1))
   2715 echo_i "flush cache ($n)"
   2716 ret=0
   2717 $RNDCCMD 10.53.0.3 flushtree example >rndc.out.test$n.1 2>&1 || ret=1
   2718 if [ $ret != 0 ]; then echo_i "failed"; fi
   2719 status=$((status + ret))
   2720 
   2721 # prime the cache with CNAME and A; CNAME expires sooner
   2722 n=$((n + 1))
   2723 echo_i "prime cache cname.example A (stale-answer-client-timeout 1.8) ($n)"
   2724 ret=0
   2725 $DIG -p ${PORT} @10.53.0.3 cname.example A >dig.out.test$n || ret=1
   2726 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2727 grep "ANSWER: 2," dig.out.test$n >/dev/null || ret=1
   2728 grep "cname\.example\..*7.*IN.*CNAME.*target\.example\." dig.out.test$n >/dev/null || ret=1
   2729 grep "target\.example\..*9.*IN.*A" dig.out.test$n >/dev/null || ret=1
   2730 if [ $ret != 0 ]; then echo_i "failed"; fi
   2731 status=$((status + ret))
   2732 
   2733 # wait for the CNAME to be stale; A will still be valid and in prefetch window.
   2734 # (the longer TTL is needed, otherwise data won't be prefetch-eligible.)
   2735 sleep 7
   2736 
   2737 # re-enable auth responses, but with a delay answering the A
   2738 n=$((n + 1))
   2739 echo_i "delay responses from authoritative server ($n)"
   2740 ret=0
   2741 $DIG -p ${PORT} @10.53.0.2 txt slowdown >dig.out.test$n || ret=1
   2742 grep "ANSWER: 1," dig.out.test$n >/dev/null || ret=1
   2743 grep "TXT.\"1\"" dig.out.test$n >/dev/null || ret=1
   2744 if [ $ret != 0 ]; then echo_i "failed"; fi
   2745 status=$((status + ret))
   2746 
   2747 # resend the query and wait in the background; we should get a stale answer
   2748 n=$((n + 1))
   2749 echo_i "check prefetch processing of a stale CNAME target ($n)"
   2750 ret=0
   2751 $DIG -p ${PORT} @10.53.0.3 cname.example A >dig.out.test$n &
   2752 sleep 2
   2753 wait
   2754 grep "status: NOERROR" dig.out.test$n >/dev/null || ret=1
   2755 grep "ANSWER: 2," dig.out.test$n >/dev/null || ret=1
   2756 grep "cname\.example\..*7.*IN.*CNAME.*target\.example\." dig.out.test$n >/dev/null || ret=1
   2757 grep "target\.example\..*[1-2].*IN.*A" dig.out.test$n >/dev/null || ret=1
   2758 if [ $ret != 0 ]; then echo_i "failed"; fi
   2759 status=$((status + ret))
   2760 
   2761 echo_i "exit status: $status"
   2762 [ $status -eq 0 ] || exit 1
   2763