Home | History | Annotate | Line # | Download | only in contrib
unbound_munin_ revision 1.1
      1 #!/bin/sh
      2 #
      3 # plugin for munin to monitor usage of unbound servers.
      4 # To install copy this to /usr/local/share/munin/plugins/unbound_munin_
      5 # and use munin-node-configure (--suggest, --shell).
      6 #
      7 # (C) 2008 W.C.A. Wijngaards.  BSD Licensed.
      8 #
      9 # To install; enable statistics and unbound-control in unbound.conf
     10 #	server:		extended-statistics: yes
     11 #			statistics-cumulative: no
     12 #			statistics-interval: 0
     13 #	remote-control:	control-enable: yes
     14 # Run the command unbound-control-setup to generate the key files.
     15 #
     16 # Environment variables for this script
     17 #	statefile	- where to put temporary statefile.
     18 #	unbound_conf	- where the unbound.conf file is located.
     19 #	unbound_control	- where to find unbound-control executable.
     20 #	spoof_warn	- what level to warn about spoofing
     21 #	spoof_crit	- what level to crit about spoofing
     22 #
     23 # You can set them in your munin/plugin-conf.d/plugins.conf file
     24 # with:
     25 # [unbound*]
     26 # user root
     27 # env.statefile /usr/local/var/munin/plugin-state/unbound-state
     28 # env.unbound_conf /usr/local/etc/unbound/unbound.conf
     29 # env.unbound_control /usr/local/sbin/unbound-control
     30 # env.spoof_warn 1000
     31 # env.spoof_crit 100000
     32 #
     33 # This plugin can create different graphs depending on what name
     34 # you link it as (with ln -s) into the plugins directory
     35 # You can link it multiple times.
     36 # If you are only a casual user, the _hits and _by_type are most interesting,
     37 # possibly followed by _by_rcode.
     38 #
     39 #	unbound_munin_hits	- base volume, cache hits, unwanted traffic
     40 #	unbound_munin_queue	- to monitor the internal requestlist
     41 #	unbound_munin_memory	- memory usage
     42 #	unbound_munin_by_type	- incoming queries by type
     43 #	unbound_munin_by_class	- incoming queries by class
     44 #	unbound_munin_by_opcode	- incoming queries by opcode
     45 #	unbound_munin_by_rcode	- answers by rcode, validation status
     46 #	unbound_munin_by_flags	- incoming queries by flags
     47 #	unbound_munin_histogram	- histogram of query resolving times
     48 #
     49 # Magic markers - optional - used by installation scripts and
     50 # munin-config:  (originally contrib family but munin-node-configure ignores it)
     51 #
     52 #%# family=auto
     53 #%# capabilities=autoconf suggest
     54 
     55 # POD documentation
     56 : <<=cut
     57 =head1 NAME
     58 
     59 unbound_munin_ - Munin plugin to monitor the Unbound DNS resolver.
     60 
     61 =head1 APPLICABLE SYSTEMS
     62 
     63 System with unbound daemon.
     64 
     65 =head1 CONFIGURATION
     66 
     67   [unbound*]
     68   user root
     69   env.statefile /usr/local/var/munin/plugin-state/unbound-state
     70   env.unbound_conf /usr/local/etc/unbound/unbound.conf
     71   env.unbound_control /usr/local/sbin/unbound-control
     72   env.spoof_warn 1000
     73   env.spoof_crit 100000
     74 
     75 Use the .env settings to override the defaults.
     76 
     77 =head1 USAGE
     78 
     79 Can be used to present different graphs. Use ln -s for that name in
     80 the plugins directory to enable the graph.
     81 unbound_munin_hits	- base volume, cache hits, unwanted traffic
     82 unbound_munin_queue	- to monitor the internal requestlist
     83 unbound_munin_memory	- memory usage
     84 unbound_munin_by_type	- incoming queries by type
     85 unbound_munin_by_class	- incoming queries by class
     86 unbound_munin_by_opcode	- incoming queries by opcode
     87 unbound_munin_by_rcode	- answers by rcode, validation status
     88 unbound_munin_by_flags	- incoming queries by flags
     89 unbound_munin_histogram - histogram of query resolving times
     90 
     91 =head1 AUTHOR
     92 
     93 Copyright 2008 W.C.A. Wijngaards
     94 
     95 =head1 LICENSE
     96 
     97 BSD
     98 
     99 =cut
    100 
    101 state=${statefile:-/usr/local/var/munin/plugin-state/unbound-state}
    102 conf=${unbound_conf:-/usr/local/etc/unbound/unbound.conf}
    103 ctrl=${unbound_control:-/usr/local/sbin/unbound-control}
    104 warn=${spoof_warn:-1000}
    105 crit=${spoof_crit:-100000}
    106 lock=$state.lock
    107 
    108 # number of seconds between polling attempts.
    109 # makes the statefile hang around for at least this many seconds,
    110 # so that multiple links of this script can share the results.
    111 lee=55
    112 
    113 # to keep things within 19 characters
    114 ABBREV="-e s/total/t/ -e s/thread/t/ -e s/num/n/ -e s/query/q/ -e s/answer/a/ -e s/unwanted/u/ -e s/requestlist/ql/ -e s/type/t/ -e s/class/c/ -e s/opcode/o/ -e s/rcode/r/ -e s/edns/e/ -e s/mem/m/ -e s/cache/c/ -e s/mod/m/"
    115 
    116 # get value from $1 into return variable $value
    117 get_value ( ) {
    118 	value="`grep '^'$1'=' $state | sed -e 's/^.*=//'`"
    119 	if test "$value"x = ""x; then
    120 		value="0"
    121 	fi
    122 }
    123 
    124 # download the state from the unbound server.
    125 get_state ( ) {
    126 	# obtain lock for fetching the state
    127 	# because there is a race condition in fetching and writing to file
    128 
    129 	# see if the lock is stale, if so, take it 
    130 	if test -f $lock ; then
    131 		pid="`cat $lock 2>&1`"
    132 		kill -0 "$pid" >/dev/null 2>&1
    133 		if test $? -ne 0 -a "$pid" != $$ ; then
    134 			echo $$ >$lock
    135 		fi
    136 	fi
    137 
    138 	i=0
    139 	while test ! -f $lock || test "`cat $lock 2>&1`" != $$; do
    140 		while test -f $lock; do
    141 			# wait
    142 			i=`expr $i + 1`
    143 			if test $i -gt 1000; then
    144 				sleep 1;
    145 			fi
    146 			if test $i -gt 1500; then
    147 				echo "error locking $lock" "=" `cat $lock`
    148 				rm -f $lock
    149 				exit 1
    150 			fi
    151 		done
    152 		# try to get it
    153 		echo $$ >$lock
    154 	done
    155 	# do not refetch if the file exists and only LEE seconds old
    156 	if test -f $state; then
    157 		now=`date +%s`
    158 		get_value "time.now"
    159 		value="`echo $value | sed -e 's/\..*$//'`"
    160 		if test $now -lt `expr $value + $lee`; then
    161 			rm -f $lock
    162 			return
    163 		fi
    164 	fi
    165 	$ctrl -c $conf stats > $state
    166 	if test $? -ne 0; then
    167 		echo "error retrieving data from unbound server"
    168 		rm -f $lock
    169 		exit 1
    170 	fi
    171 	rm -f $lock
    172 }
    173 
    174 if test "$1" = "autoconf" ; then
    175 	if test ! -f $conf; then
    176 		echo no "($conf does not exist)"
    177 		exit 1
    178 	fi
    179 	if test ! -d `dirname $state`; then
    180 		echo no "(`dirname $state` directory does not exist)"
    181 		exit 1
    182 	fi
    183 	echo yes
    184 	exit 0
    185 fi
    186 
    187 if test "$1" = "suggest" ; then
    188 	echo "hits"
    189 	echo "queue"
    190 	echo "memory"
    191 	echo "by_type"
    192 	echo "by_class"
    193 	echo "by_opcode"
    194 	echo "by_rcode"
    195 	echo "by_flags"
    196 	echo "histogram"
    197 	exit 0
    198 fi
    199 
    200 # determine my type, by name
    201 id=`echo $0 | sed -e 's/^.*unbound_munin_//'`
    202 if test "$id"x = ""x; then
    203 	# some default to keep people sane.
    204 	id="hits"
    205 fi
    206 
    207 # if $1 exists in statefile, config is echoed with label $2
    208 exist_config ( ) {
    209 	mn=`echo $1 | sed $ABBREV | tr . _`
    210 	if grep '^'$1'=' $state >/dev/null 2>&1; then
    211 		echo "$mn.label $2"
    212 		echo "$mn.min 0"
    213 		echo "$mn.type ABSOLUTE"
    214 	fi
    215 }
    216 
    217 # print label and min 0 for a name $1 in unbound format
    218 p_config ( ) {
    219 	mn=`echo $1 | sed $ABBREV | tr . _`
    220 	echo $mn.label "$2"
    221 	echo $mn.min 0
    222 	echo $mn.type $3
    223 }
    224 
    225 if test "$1" = "config" ; then
    226 	if test ! -f $state; then
    227 		get_state
    228 	fi
    229 	case $id in
    230 	hits)
    231 		echo "graph_title Unbound DNS traffic and cache hits"
    232 		echo "graph_args --base 1000 -l 0"
    233 		echo "graph_vlabel queries / \${graph_period}"
    234 		echo "graph_scale no"
    235 		echo "graph_category DNS"
    236 		for x in `grep "^thread[0-9][0-9]*\.num\.queries=" $state |
    237 			sed -e 's/=.*//'`; do
    238 			exist_config $x "queries handled by `basename $x .num.queries`"
    239 		done
    240 		p_config "total.num.queries" "total queries from clients" "ABSOLUTE"
    241 		p_config "total.num.cachehits" "cache hits" "ABSOLUTE"
    242 		p_config "total.num.prefetch" "cache prefetch" "ABSOLUTE"
    243 		p_config "num.query.tcp" "TCP queries" "ABSOLUTE"
    244 		p_config "num.query.tcpout" "TCP out queries" "ABSOLUTE"
    245 		p_config "num.query.ipv6" "IPv6 queries" "ABSOLUTE"
    246 		p_config "unwanted.queries" "queries that failed acl" "ABSOLUTE"
    247 		p_config "unwanted.replies" "unwanted or unsolicited replies" "ABSOLUTE"
    248 		echo "u_replies.warning $warn"
    249 		echo "u_replies.critical $crit"
    250 		echo "graph_info DNS queries to the recursive resolver. The unwanted replies could be innocent duplicate packets, late replies, or spoof threats."
    251 		;;
    252 	queue)
    253 		echo "graph_title Unbound requestlist size"
    254 		echo "graph_args --base 1000 -l 0"
    255 		echo "graph_vlabel number of queries"
    256 		echo "graph_scale no"
    257 		echo "graph_category DNS"
    258 		p_config "total.requestlist.avg" "Average size of queue on insert" "GAUGE"
    259 		p_config "total.requestlist.max" "Max size of queue (in 5 min)" "GAUGE"
    260 		p_config "total.requestlist.overwritten" "Number of queries replaced by new ones" "GAUGE"
    261 		p_config "total.requestlist.exceeded" "Number of queries dropped due to lack of space" "GAUGE"
    262 		echo "graph_info The queries that did not hit the cache and need recursion service take up space in the requestlist. If there are too many queries, first queries get overwritten, and at last resort dropped."
    263 		;;
    264 	memory)
    265 		echo "graph_title Unbound memory usage"
    266 		echo "graph_args --base 1024 -l 0"
    267 		echo "graph_vlabel memory used in bytes"
    268 		echo "graph_category DNS"
    269 		p_config "mem.total.sbrk" "Total memory" "GAUGE"
    270 		p_config "mem.cache.rrset" "RRset cache memory" "GAUGE"
    271 		p_config "mem.cache.message" "Message cache memory" "GAUGE"
    272 		p_config "mem.mod.iterator" "Iterator module memory" "GAUGE"
    273 		p_config "mem.mod.validator" "Validator module and key cache memory" "GAUGE"
    274 		p_config "msg.cache.count" "msg cache count" "GAUGE"
    275 		p_config "rrset.cache.count" "rrset cache count" "GAUGE"
    276 		p_config "infra.cache.count" "infra cache count" "GAUGE"
    277 		p_config "key.cache.count" "key cache count" "GAUGE"
    278 		echo "graph_info The memory used by unbound."
    279 		;;
    280 	by_type)
    281 		echo "graph_title Unbound DNS queries by type"
    282 		echo "graph_args --base 1000 -l 0"
    283 		echo "graph_vlabel queries / \${graph_period}"
    284 		echo "graph_scale no"
    285 		echo "graph_category DNS"
    286 		for x in `grep "^num.query.type" $state`; do
    287 			nm=`echo $x | sed -e 's/=.*$//'`
    288 			tp=`echo $nm | sed -e s/num.query.type.//`
    289 			p_config "$nm" "$tp" "ABSOLUTE"
    290 		done
    291 		echo "graph_info queries by DNS RR type queried for"
    292 		;;
    293 	by_class)
    294 		echo "graph_title Unbound DNS queries by class"
    295 		echo "graph_args --base 1000 -l 0"
    296 		echo "graph_vlabel queries / \${graph_period}"
    297 		echo "graph_scale no"
    298 		echo "graph_category DNS"
    299 		for x in `grep "^num.query.class" $state`; do
    300 			nm=`echo $x | sed -e 's/=.*$//'`
    301 			tp=`echo $nm | sed -e s/num.query.class.//`
    302 			p_config "$nm" "$tp" "ABSOLUTE"
    303 		done
    304 		echo "graph_info queries by DNS RR class queried for."
    305 		;;
    306 	by_opcode)
    307 		echo "graph_title Unbound DNS queries by opcode"
    308 		echo "graph_args --base 1000 -l 0"
    309 		echo "graph_vlabel queries / \${graph_period}"
    310 		echo "graph_scale no"
    311 		echo "graph_category DNS"
    312 		for x in `grep "^num.query.opcode" $state`; do
    313 			nm=`echo $x | sed -e 's/=.*$//'`
    314 			tp=`echo $nm | sed -e s/num.query.opcode.//`
    315 			p_config "$nm" "$tp" "ABSOLUTE"
    316 		done
    317 		echo "graph_info queries by opcode in the query packet."
    318 		;;
    319 	by_rcode)
    320 		echo "graph_title Unbound DNS answers by return code"
    321 		echo "graph_args --base 1000 -l 0"
    322 		echo "graph_vlabel answer packets / \${graph_period}"
    323 		echo "graph_scale no"
    324 		echo "graph_category DNS"
    325 		for x in `grep "^num.answer.rcode" $state`; do
    326 			nm=`echo $x | sed -e 's/=.*$//'`
    327 			tp=`echo $nm | sed -e s/num.answer.rcode.//`
    328 			p_config "$nm" "$tp" "ABSOLUTE"
    329 		done
    330 		p_config "num.answer.secure" "answer secure" "ABSOLUTE"
    331 		p_config "num.answer.bogus" "answer bogus" "ABSOLUTE"
    332 		p_config "num.rrset.bogus" "num rrsets marked bogus" "ABSOLUTE"
    333 		echo "graph_info answers sorted by return value. rrsets bogus is the number of rrsets marked bogus per \${graph_period} by the validator"
    334 		;;
    335 	by_flags)
    336 		echo "graph_title Unbound DNS incoming queries by flags"
    337 		echo "graph_args --base 1000 -l 0"
    338 		echo "graph_vlabel queries / \${graph_period}"
    339 		echo "graph_scale no"
    340 		echo "graph_category DNS"
    341 		p_config "num.query.flags.QR" "QR (query reply) flag" "ABSOLUTE"
    342 		p_config "num.query.flags.AA" "AA (auth answer) flag" "ABSOLUTE"
    343 		p_config "num.query.flags.TC" "TC (truncated) flag" "ABSOLUTE"
    344 		p_config "num.query.flags.RD" "RD (recursion desired) flag" "ABSOLUTE"
    345 		p_config "num.query.flags.RA" "RA (rec avail) flag" "ABSOLUTE"
    346 		p_config "num.query.flags.Z" "Z (zero) flag" "ABSOLUTE"
    347 		p_config "num.query.flags.AD" "AD (auth data) flag" "ABSOLUTE"
    348 		p_config "num.query.flags.CD" "CD (check disabled) flag" "ABSOLUTE"
    349 		p_config "num.query.edns.present" "EDNS OPT present" "ABSOLUTE"
    350 		p_config "num.query.edns.DO" "DO (DNSSEC OK) flag" "ABSOLUTE"
    351 		echo "graph_info This graphs plots the flags inside incoming queries. For example, if QR, AA, TC, RA, Z flags are set, the query can be rejected. RD, AD, CD and DO are legitimately set by some software."
    352 		;;
    353 	histogram)
    354 		echo "graph_title Unbound DNS histogram of reply time"
    355 		echo "graph_args --base 1000 -l 0"
    356 		echo "graph_vlabel queries / \${graph_period}"
    357 		echo "graph_scale no"
    358 		echo "graph_category DNS"
    359 		echo hcache.label "cache hits"
    360 		echo hcache.min 0
    361 		echo hcache.type ABSOLUTE
    362 		echo hcache.draw AREA
    363 		echo hcache.colour 999999
    364 		echo h64ms.label "0 msec - 66 msec"
    365 		echo h64ms.min 0
    366 		echo h64ms.type ABSOLUTE
    367 		echo h64ms.draw STACK
    368 		echo h64ms.colour 0000FF
    369 		echo h128ms.label "66 msec - 131 msec"
    370 		echo h128ms.min 0
    371 		echo h128ms.type ABSOLUTE
    372 		echo h128ms.colour 1F00DF
    373 		echo h128ms.draw STACK
    374 		echo h256ms.label "131 msec - 262 msec"
    375 		echo h256ms.min 0
    376 		echo h256ms.type ABSOLUTE
    377 		echo h256ms.draw STACK
    378 		echo h256ms.colour 3F00BF
    379 		echo h512ms.label "262 msec - 524 msec"
    380 		echo h512ms.min 0
    381 		echo h512ms.type ABSOLUTE
    382 		echo h512ms.draw STACK
    383 		echo h512ms.colour 5F009F
    384 		echo h1s.label "524 msec - 1 sec"
    385 		echo h1s.min 0
    386 		echo h1s.type ABSOLUTE
    387 		echo h1s.draw STACK
    388 		echo h1s.colour 7F007F
    389 		echo h2s.label "1 sec - 2 sec"
    390 		echo h2s.min 0
    391 		echo h2s.type ABSOLUTE
    392 		echo h2s.draw STACK
    393 		echo h2s.colour 9F005F
    394 		echo h4s.label "2 sec - 4 sec"
    395 		echo h4s.min 0
    396 		echo h4s.type ABSOLUTE
    397 		echo h4s.draw STACK
    398 		echo h4s.colour BF003F
    399 		echo h8s.label "4 sec - 8 sec"
    400 		echo h8s.min 0
    401 		echo h8s.type ABSOLUTE
    402 		echo h8s.draw STACK
    403 		echo h8s.colour DF001F
    404 		echo h16s.label "8 sec - ..."
    405 		echo h16s.min 0
    406 		echo h16s.type ABSOLUTE
    407 		echo h16s.draw STACK
    408 		echo h16s.colour FF0000
    409 		echo "graph_info Histogram of the reply times for queries."
    410 		;;
    411 	esac
    412 
    413 	exit 0
    414 fi
    415 
    416 # do the stats itself
    417 get_state
    418 
    419 # get the time elapsed
    420 get_value "time.elapsed"
    421 if test $value = 0 || test $value = "0.000000"; then
    422 	echo "error: time elapsed 0 or could not retrieve data"
    423 	exit 1
    424 fi
    425 elapsed="$value"
    426 
    427 # print value for $1
    428 print_value ( ) {
    429 	mn=`echo $1 | sed $ABBREV | tr . _`
    430 	get_value $1
    431 	echo "$mn.value" $value
    432 }
    433 
    434 # print value if line already found in $2
    435 print_value_line ( ) {
    436 	mn=`echo $1 | sed $ABBREV | tr . _`
    437 	value="`echo $2 | sed -e 's/^.*=//'`"
    438 	echo "$mn.value" $value
    439 }
    440 
    441 
    442 case $id in
    443 hits)
    444 	for x in `grep "^thread[0-9][0-9]*\.num\.queries=" $state |
    445 		sed -e 's/=.*//'` total.num.queries \
    446 		total.num.cachehits total.num.prefetch num.query.tcp \
    447 		num.query.tcpout num.query.ipv6 unwanted.queries \
    448 		unwanted.replies; do
    449 		if grep "^"$x"=" $state >/dev/null 2>&1; then
    450 			print_value $x
    451 		fi
    452 	done
    453 	;;
    454 queue)
    455 	for x in total.requestlist.avg total.requestlist.max \
    456 		total.requestlist.overwritten total.requestlist.exceeded; do
    457 		print_value $x
    458 	done
    459 	;;
    460 memory)
    461 	mn=`echo mem.total.sbrk | sed $ABBREV | tr . _`
    462 	get_value 'mem.total.sbrk'
    463 	if test $value -eq 0; then
    464 		chk=`echo $ctrl | sed -e 's/-control$/-checkconf/'`
    465 		pidf=`$chk -o pidfile $conf 2>&1`
    466 		pid=`cat $pidf 2>&1`
    467 		value=`ps -p "$pid" -o rss= 2>&1`
    468 		if test "`expr $value + 1 - 1 2>&1`" -eq "$value" 2>&1; then
    469 			value=`expr $value \* 1024` 
    470 		else
    471 			value=0
    472 		fi
    473 	fi
    474 	echo "$mn.value" $value
    475 	for x in mem.cache.rrset mem.cache.message mem.mod.iterator \
    476 		mem.mod.validator msg.cache.count rrset.cache.count \
    477 		infra.cache.count key.cache.count; do
    478 		print_value $x
    479 	done
    480 	;;
    481 by_type)
    482 	for x in `grep "^num.query.type" $state`; do
    483 		nm=`echo $x | sed -e 's/=.*$//'`
    484 		print_value_line $nm $x
    485 	done
    486 	;;
    487 by_class)
    488 	for x in `grep "^num.query.class" $state`; do
    489 		nm=`echo $x | sed -e 's/=.*$//'`
    490 		print_value_line $nm $x
    491 	done
    492 	;;
    493 by_opcode)
    494 	for x in `grep "^num.query.opcode" $state`; do
    495 		nm=`echo $x | sed -e 's/=.*$//'`
    496 		print_value_line $nm $x
    497 	done
    498 	;;
    499 by_rcode)
    500 	for x in `grep "^num.answer.rcode" $state`; do
    501 		nm=`echo $x | sed -e 's/=.*$//'`
    502 		print_value_line $nm $x
    503 	done
    504 	print_value "num.answer.secure"
    505 	print_value "num.answer.bogus"
    506 	print_value "num.rrset.bogus"
    507 	;;
    508 by_flags)
    509 	for x in num.query.flags.QR num.query.flags.AA num.query.flags.TC num.query.flags.RD num.query.flags.RA num.query.flags.Z num.query.flags.AD num.query.flags.CD num.query.edns.present num.query.edns.DO; do
    510 		print_value $x
    511 	done
    512 	;;
    513 histogram)
    514 	get_value total.num.cachehits
    515 	echo hcache.value $value
    516 	r=0
    517 	for x in histogram.000000.000000.to.000000.000001 \
    518 		histogram.000000.000001.to.000000.000002 \
    519 		histogram.000000.000002.to.000000.000004 \
    520 		histogram.000000.000004.to.000000.000008 \
    521 		histogram.000000.000008.to.000000.000016 \
    522 		histogram.000000.000016.to.000000.000032 \
    523 		histogram.000000.000032.to.000000.000064 \
    524 		histogram.000000.000064.to.000000.000128 \
    525 		histogram.000000.000128.to.000000.000256 \
    526 		histogram.000000.000256.to.000000.000512 \
    527 		histogram.000000.000512.to.000000.001024 \
    528 		histogram.000000.001024.to.000000.002048 \
    529 		histogram.000000.002048.to.000000.004096 \
    530 		histogram.000000.004096.to.000000.008192 \
    531 		histogram.000000.008192.to.000000.016384 \
    532 		histogram.000000.016384.to.000000.032768 \
    533 		histogram.000000.032768.to.000000.065536; do
    534 		get_value $x
    535 		r=`expr $r + $value`
    536 	done
    537 	echo h64ms.value $r
    538 	get_value histogram.000000.065536.to.000000.131072
    539 	echo h128ms.value $value
    540 	get_value histogram.000000.131072.to.000000.262144
    541 	echo h256ms.value $value
    542 	get_value histogram.000000.262144.to.000000.524288
    543 	echo h512ms.value $value
    544 	get_value histogram.000000.524288.to.000001.000000
    545 	echo h1s.value $value
    546 	get_value histogram.000001.000000.to.000002.000000
    547 	echo h2s.value $value
    548 	get_value histogram.000002.000000.to.000004.000000
    549 	echo h4s.value $value
    550 	get_value histogram.000004.000000.to.000008.000000
    551 	echo h8s.value $value
    552 	r=0
    553 	for x in histogram.000008.000000.to.000016.000000 \
    554 		histogram.000016.000000.to.000032.000000 \
    555 		histogram.000032.000000.to.000064.000000 \
    556 		histogram.000064.000000.to.000128.000000 \
    557 		histogram.000128.000000.to.000256.000000 \
    558 		histogram.000256.000000.to.000512.000000 \
    559 		histogram.000512.000000.to.001024.000000 \
    560 		histogram.001024.000000.to.002048.000000 \
    561 		histogram.002048.000000.to.004096.000000 \
    562 		histogram.004096.000000.to.008192.000000 \
    563 		histogram.008192.000000.to.016384.000000 \
    564 		histogram.016384.000000.to.032768.000000 \
    565 		histogram.032768.000000.to.065536.000000 \
    566 		histogram.065536.000000.to.131072.000000 \
    567 		histogram.131072.000000.to.262144.000000 \
    568 		histogram.262144.000000.to.524288.000000; do
    569 		get_value $x
    570 		r=`expr $r + $value`
    571 	done
    572 	echo h16s.value $r
    573 	;;
    574 esac
    575