Home | History | Annotate | Line # | Download | only in fetchlimit
      1 #!/bin/sh
      2 
      3 # Copyright (C) Internet Systems Consortium, Inc. ("ISC")
      4 #
      5 # SPDX-License-Identifier: MPL-2.0
      6 #
      7 # This Source Code Form is subject to the terms of the Mozilla Public
      8 # License, v. 2.0.  If a copy of the MPL was not distributed with this
      9 # file, you can obtain one at https://mozilla.org/MPL/2.0/.
     10 #
     11 # See the COPYRIGHT file distributed with this work for additional
     12 # information regarding copyright ownership.
     13 
     14 set -e
     15 
     16 . ../conf.sh
     17 
     18 DIGCMD="$DIG @10.53.0.3 -p ${PORT} +tcp +tries=1 +time=1"
     19 
     20 rndccmd() (
     21   "$RNDC" -c ../_common/rndc.conf -p "${CONTROLPORT}" -s "$@"
     22 )
     23 
     24 dig_with_opts() (
     25   "$DIG" -p "$PORT" "$@"
     26 )
     27 
     28 sendcmd() (
     29   SERVER="${1}"
     30   COMMAND="${2}"
     31   COMMAND_ARGS="${3}"
     32   dig_with_opts "@${SERVER}" "${COMMAND_ARGS}.${COMMAND}._control." TXT +time=5 +tries=1 +tcp >/dev/null 2>&1
     33 )
     34 
     35 burst() {
     36   server=${1}
     37   num=${4:-20}
     38   rm -f burst.input.$$
     39   while [ $num -gt 0 ]; do
     40     num=$((num - 1))
     41     if [ "${5}" = "dup" ]; then
     42       # burst with duplicate queries
     43       echo "${2}${3}.lamesub.example A" >>burst.input.$$
     44     else
     45       # burst with unique queries
     46       echo "${num}${2}${3}.lamesub.example A" >>burst.input.$$
     47     fi
     48   done
     49   $PERL ../ditch.pl -p ${PORT} -s ${server} -b ${EXTRAPORT8} burst.input.$$
     50   rm -f burst.input.$$
     51 }
     52 
     53 stat() {
     54   clients=$(rndccmd ${1} status | grep "recursive clients" \
     55     | sed 's;.*: \([^/][^/]*\)/.*;\1;')
     56   echo_i "clients: $clients"
     57   [ "$clients" = "" ] && return 1
     58   [ "$clients" -ge $2 ] || return 1
     59   [ "$clients" -le $3 ] || return 1
     60   return 0
     61 }
     62 
     63 _wait_for_message() (
     64   nextpartpeek "$1" >wait_for_message.$n
     65   grep -F "$2" wait_for_message.$n >/dev/null
     66 )
     67 
     68 wait_for_message() (
     69   retry_quiet 20 _wait_for_message "$@"
     70 )
     71 
     72 n=0
     73 status=0
     74 
     75 n=$((n + 1))
     76 echo_i "checking recursing clients are dropped at the per-server limit ($n)"
     77 ret=0
     78 # make the server lame and restart
     79 rndccmd 10.53.0.3 flush
     80 sendcmd 10.53.0.4 send-responses "disable"
     81 for try in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do
     82   burst 10.53.0.3 a $try
     83   # fetches-per-server is at 400, but at 20qps against a lame server,
     84   # we'll reach 200 at the tenth second, and the quota should have been
     85   # tuned to less than that by then.
     86   [ $try -le 5 ] && low=$((try * 10))
     87   stat 10.53.0.3 20 200 || ret=1
     88   [ $ret -eq 1 ] && break
     89   sleep 1
     90 done
     91 if [ $ret != 0 ]; then echo_i "failed"; fi
     92 status=$((status + ret))
     93 
     94 n=$((n + 1))
     95 echo_i "dumping ADB data ($n)"
     96 ret=0
     97 info=$(rndccmd 10.53.0.3 fetchlimit | grep 10.53.0.4 | sed 's/.*quota .*(\([0-9]*\).*atr \([.0-9]*\).*/\2 \1/')
     98 echo_i $info
     99 set -- $info
    100 quota=$2
    101 [ ${quota:-200} -lt 200 ] || ret=1
    102 if [ $ret != 0 ]; then echo_i "failed"; fi
    103 status=$((status + ret))
    104 
    105 n=$((n + 1))
    106 echo_i "checking servfail statistics ($n)"
    107 ret=0
    108 rm -f ns3/named.stats
    109 rndccmd 10.53.0.3 stats
    110 for try in 1 2 3 4 5; do
    111   [ -f ns3/named.stats ] && break
    112   sleep 1
    113 done
    114 sspill=$(grep 'spilled due to server' ns3/named.stats | sed 's/\([0-9][0-9]*\) spilled.*/\1/')
    115 [ -z "$sspill" ] && sspill=0
    116 fails=$(grep 'queries resulted in SERVFAIL' ns3/named.stats | sed 's/\([0-9][0-9]*\) queries.*/\1/')
    117 [ -z "$fails" ] && fails=0
    118 [ "$fails" -ge "$sspill" ] || ret=1
    119 if [ $ret != 0 ]; then echo_i "failed"; fi
    120 status=$((status + ret))
    121 
    122 n=$((n + 1))
    123 echo_i "checking lame server recovery ($n)"
    124 ret=0
    125 sendcmd 10.53.0.4 send-responses "enable"
    126 for try in 1 2 3 4 5; do
    127   burst 10.53.0.3 b $try
    128   stat 10.53.0.3 0 200 || ret=1
    129   [ $ret -eq 1 ] && break
    130   sleep 1
    131 done
    132 if [ $ret != 0 ]; then echo_i "failed"; fi
    133 status=$((status + ret))
    134 
    135 n=$((n + 1))
    136 echo_i "dumping ADB data ($n)"
    137 ret=0
    138 info=$(rndccmd 10.53.0.3 fetchlimit | grep 10.53.0.4 | sed 's/.*quota .*(\([0-9]*\).*atr \([.0-9]*\).*/\2 \1/')
    139 echo_i $info
    140 set -- $info
    141 [ ${2:-${quota}} -lt $quota ] || ret=1
    142 quota=$2
    143 if [ $ret != 0 ]; then echo_i "failed"; fi
    144 status=$((status + ret))
    145 
    146 n=$((n + 1))
    147 echo_i "checking lame server recovery (continued) ($n)"
    148 ret=0
    149 for try in 1 2 3 4 5 6 7 8 9 10; do
    150   burst 10.53.0.3 c $try
    151   stat 10.53.0.3 0 20 || ret=1
    152   [ $ret -eq 1 ] && break
    153   sleep 1
    154 done
    155 if [ $ret != 0 ]; then echo_i "failed"; fi
    156 status=$((status + ret))
    157 
    158 n=$((n + 1))
    159 echo_i "dumping ADB data ($n)"
    160 ret=0
    161 info=$(rndccmd 10.53.0.3 fetchlimit | grep 10.53.0.4 | sed 's/.*quota .*(\([0-9]*\).*atr \([.0-9]*\).*/\2 \1/')
    162 echo_i $info
    163 set -- $info
    164 [ ${2:-${quota}} -gt $quota ] || ret=1
    165 quota=$2
    166 if [ $ret != 0 ]; then echo_i "failed"; fi
    167 status=$((status + ret))
    168 
    169 cp ns3/named2.conf ns3/named.conf
    170 rndc_reconfig ns3 10.53.0.3
    171 
    172 n=$((n + 1))
    173 echo_i "checking lame server clients are dropped at the per-domain limit ($n)"
    174 ret=0
    175 fail=0
    176 success=0
    177 sendcmd 10.53.0.4 send-responses "disable"
    178 for try in 1 2 3 4 5; do
    179   burst 10.53.0.3 d $try 300
    180   $DIGCMD a ${try}.example >dig.out.ns3.$n.$try
    181   grep "status: NOERROR" dig.out.ns3.$n.$try >/dev/null 2>&1 \
    182     && success=$((success + 1))
    183   grep "status: SERVFAIL" dig.out.ns3.$n.$try >/dev/null 2>&1 \
    184     && fail=$(($fail + 1))
    185   stat 10.53.0.3 40 40 || ret=1
    186   allowed=$(rndccmd 10.53.0.3 fetchlimit | awk '/lamesub/ { print $6 }')
    187   [ "${allowed:-0}" -eq 40 ] || ret=1
    188   [ $ret -eq 1 ] && break
    189   sleep 1
    190 done
    191 echo_i "$success successful valid queries, $fail SERVFAIL"
    192 if [ $ret != 0 ]; then echo_i "failed"; fi
    193 status=$((status + ret))
    194 
    195 n=$((n + 1))
    196 echo_i "checking drop statistics ($n)"
    197 ret=0
    198 rm -f ns3/named.stats
    199 rndccmd 10.53.0.3 stats
    200 for try in 1 2 3 4 5; do
    201   [ -f ns3/named.stats ] && break
    202   sleep 1
    203 done
    204 zspill=$(grep 'spilled due to zone' ns3/named.stats | sed 's/\([0-9][0-9]*\) spilled.*/\1/')
    205 [ -z "$zspill" ] && zspill=0
    206 drops=$(grep 'queries dropped' ns3/named.stats | sed 's/\([0-9][0-9]*\) queries.*/\1/')
    207 [ -z "$drops" ] && drops=0
    208 [ "$drops" -ge "$zspill" ] || ret=1
    209 if [ $ret != 0 ]; then echo_i "failed"; fi
    210 status=$((status + ret))
    211 
    212 cp ns3/named3.conf ns3/named.conf
    213 rndc_reconfig ns3 10.53.0.3
    214 
    215 n=$((n + 1))
    216 echo_i "checking lame server clients are dropped below the hard limit ($n)"
    217 ret=0
    218 fail=0
    219 exceeded=0
    220 success=0
    221 sendcmd 10.53.0.4 send-responses "disable"
    222 for try in 1 2 3 4 5; do
    223   burst 10.53.0.3 b $try 400
    224   $DIGCMD +time=2 a ${try}.example >dig.out.ns3.$n.$try
    225   stat 10.53.0.3 1 400 || exceeded=$((exceeded + 1))
    226   grep "status: NOERROR" dig.out.ns3.$n.$try >/dev/null 2>&1 \
    227     && success=$((success + 1))
    228   grep "status: SERVFAIL" dig.out.ns3.$n.$try >/dev/null 2>&1 \
    229     && fail=$(($fail + 1))
    230   sleep 1
    231 done
    232 echo_i "$success successful valid queries (expected 5)"
    233 [ "$success" -eq 5 ] || {
    234   echo_i "failed"
    235   ret=1
    236 }
    237 echo_i "$fail SERVFAIL responses (expected 0)"
    238 [ "$fail" -eq 0 ] || {
    239   echo_i "failed"
    240   ret=1
    241 }
    242 echo_i "clients count exceeded 400 on $exceeded trials (expected 0)"
    243 [ "$exceeded" -eq 0 ] || {
    244   echo_i "failed"
    245   ret=1
    246 }
    247 if [ $ret != 0 ]; then echo_i "failed"; fi
    248 status=$((status + ret))
    249 
    250 n=$((n + 1))
    251 echo_i "checking drop statistics ($n)"
    252 ret=0
    253 rm -f ns3/named.stats
    254 touch ns3/named.stats
    255 rndccmd 10.53.0.3 stats
    256 wait_for_log 5 "queries dropped due to recursive client limit" ns3/named.stats || ret=1
    257 drops=$(grep 'queries dropped due to recursive client limit' ns3/named.stats | sed 's/\([0-9][0-9]*\) queries.*/\1/')
    258 [ "${drops:-0}" -ne 0 ] || ret=1
    259 if [ $ret != 0 ]; then echo_i "failed"; fi
    260 status=$((status + ret))
    261 
    262 nextpart ns5/named.run >/dev/null
    263 
    264 n=$((n + 1))
    265 echo_i "checking clients are dropped at the clients-per-query limit ($n)"
    266 ret=0
    267 sendcmd 10.53.0.4 send-responses "enable"
    268 for try in 1 2 3 4 5; do
    269   burst 10.53.0.5 latency $try 20 "dup"
    270   sleep 1
    271 done
    272 wait_for_message ns5/named.run "clients-per-query increased to 10" || ret=1
    273 if [ $ret != 0 ]; then echo_i "failed"; fi
    274 status=$((status + ret))
    275 
    276 n=$((n + 1))
    277 echo_i "checking drop statistics ($n)"
    278 ret=0
    279 rm -f ns5/named.stats
    280 rndccmd 10.53.0.5 stats
    281 for try in 1 2 3 4 5; do
    282   [ -f ns5/named.stats ] && break
    283   sleep 1
    284 done
    285 zspill=$(grep 'spilled due to clients per query' ns5/named.stats | sed 's/ *\([0-9][0-9]*\) spilled.*/\1/')
    286 [ -z "$zspill" ] && zspill=0
    287 # ns5 configuration:
    288 #   clients-per-query 5
    289 #   max-clients-per-query 10
    290 # expected spills:
    291 #   15 (out of 20) spilled for the first burst, and 10 (out of 20) spilled for
    292 #   the next 4 bursts (because of auto-tuning): 15 + (4 * 10) == 55
    293 expected=55
    294 [ "$zspill" -eq "$expected" ] || ret=1
    295 echo_i "$zspill clients spilled (expected $expected)"
    296 if [ $ret != 0 ]; then echo_i "failed"; fi
    297 status=$((status + ret))
    298 
    299 echo_i "stop ns5"
    300 stop_server --use-rndc --port ${CONTROLPORT} ns5
    301 cp ns5/named2.conf ns5/named.conf
    302 echo_i "start ns5"
    303 start_server --noclean --restart --port ${PORT} ns5
    304 
    305 nextpart ns5/named.run >/dev/null
    306 
    307 n=$((n + 1))
    308 echo_i "checking clients are dropped at the clients-per-query limit with stale-answer-client-timeout ($n)"
    309 ret=0
    310 sendcmd 10.53.0.4 send-responses "enable"
    311 for try in 1 2 3 4 5; do
    312   burst 10.53.0.5 latency $try 20 "dup"
    313   sleep 1
    314 done
    315 wait_for_message ns5/named.run "clients-per-query increased to 10" || ret=1
    316 if [ $ret != 0 ]; then echo_i "failed"; fi
    317 status=$((status + ret))
    318 
    319 n=$((n + 1))
    320 echo_i "checking drop statistics ($n)"
    321 ret=0
    322 rm -f ns5/named.stats
    323 rndccmd 10.53.0.5 stats
    324 for try in 1 2 3 4 5; do
    325   [ -f ns5/named.stats ] && break
    326   sleep 1
    327 done
    328 zspill=$(grep 'spilled due to clients per query' ns5/named.stats | sed 's/ *\([0-9][0-9]*\) spilled.*/\1/')
    329 [ -z "$zspill" ] && zspill=0
    330 # ns5 configuration:
    331 #   clients-per-query 5
    332 #   max-clients-per-query 10
    333 # expected spills:
    334 #   15 (out of 20) spilled for the first burst, and 10 (out of 20) spilled for
    335 #   the next 4 bursts (because of auto-tuning): 15 + (4 * 10) == 55
    336 expected=55
    337 [ "$zspill" -eq "$expected" ] || ret=1
    338 echo_i "$zspill clients spilled (expected $expected)"
    339 if [ $ret != 0 ]; then echo_i "failed"; fi
    340 status=$((status + ret))
    341 
    342 n=$((n + 1))
    343 echo_i "checking a warning is logged if max-clients-per-query < clients-per-query ($n)"
    344 ret=0
    345 cp ns5/named3.conf ns5/named.conf
    346 rndc_reconfig ns5 10.53.0.5
    347 wait_for_message ns5/named.run "configured clients-per-query (10) exceeds max-clients-per-query (5); automatically adjusting max-clients-per-query to (10)" || ret=1
    348 if [ $ret != 0 ]; then echo_i "failed"; fi
    349 status=$((status + ret))
    350 
    351 echo_i "exit status: $status"
    352 [ $status -eq 0 ] || exit 1
    353