Home | History | Annotate | Line # | Download | only in unit-tests
Makefile revision 1.139
      1 # $NetBSD: Makefile,v 1.139 2020/09/13 09:17:47 rillig Exp $
      2 #
      3 # Unit tests for make(1)
      4 #
      5 # The main targets are:
      6 #
      7 # all:
      8 #	run all the tests
      9 # test:
     10 #	run 'all', and compare to expected results
     11 # accept:
     12 #	move generated output to expected results
     13 #
     14 # Settable variables
     15 #
     16 # TEST_MAKE
     17 #	The make program to be tested.
     18 #
     19 #
     20 # Adding a test case
     21 #
     22 # Each feature should get its own set of tests in its own suitably
     23 # named makefile (*.mk), with its own set of expected results (*.exp),
     24 # and it should be added to the TESTS list.
     25 #
     26 # Any added files must also be added to src/distrib/sets/lists/tests/mi.
     27 # To do that, just run "make sync-mi" in this directory.
     28 #
     29 # A few *.mk files are helper files for other tests (such as include-sub.mk)
     30 # and are thus not added to TESTS.  Such files must be ignored in
     31 # src/tests/usr.bin/make/t_make.sh.
     32 #
     33 
     34 # Each test is in a sub-makefile.
     35 # Keep the list sorted.
     36 # Any test that is commented out must be ignored in
     37 # src/tests/usr.bin/make/t_make.sh as well.
     38 TESTS+=		archive
     39 TESTS+=		archive-suffix
     40 TESTS+=		cmd-interrupt
     41 TESTS+=		cmdline
     42 TESTS+=		comment
     43 TESTS+=		cond-cmp-numeric
     44 TESTS+=		cond-cmp-numeric-eq
     45 TESTS+=		cond-cmp-numeric-ge
     46 TESTS+=		cond-cmp-numeric-gt
     47 TESTS+=		cond-cmp-numeric-le
     48 TESTS+=		cond-cmp-numeric-lt
     49 TESTS+=		cond-cmp-numeric-ne
     50 TESTS+=		cond-cmp-string
     51 TESTS+=		cond-func
     52 TESTS+=		cond-func-commands
     53 TESTS+=		cond-func-defined
     54 TESTS+=		cond-func-empty
     55 TESTS+=		cond-func-exists
     56 TESTS+=		cond-func-make
     57 TESTS+=		cond-func-target
     58 TESTS+=		cond-late
     59 TESTS+=		cond-op
     60 TESTS+=		cond-op-and
     61 TESTS+=		cond-op-not
     62 TESTS+=		cond-op-or
     63 TESTS+=		cond-op-parentheses
     64 TESTS+=		cond-short
     65 TESTS+=		cond-token-number
     66 TESTS+=		cond-token-plain
     67 TESTS+=		cond-token-string
     68 TESTS+=		cond-token-var
     69 TESTS+=		cond1
     70 TESTS+=		cond2
     71 TESTS+=		counter
     72 TESTS+=		dep
     73 TESTS+=		dep-colon
     74 TESTS+=		dep-double-colon
     75 TESTS+=		dep-exclam
     76 TESTS+=		dep-none
     77 TESTS+=		dep-var
     78 TESTS+=		dep-wildcards
     79 TESTS+=		depsrc
     80 TESTS+=		depsrc-exec
     81 TESTS+=		depsrc-ignore
     82 TESTS+=		depsrc-made
     83 TESTS+=		depsrc-make
     84 TESTS+=		depsrc-meta
     85 TESTS+=		depsrc-nometa
     86 TESTS+=		depsrc-nometa_cmp
     87 TESTS+=		depsrc-nopath
     88 TESTS+=		depsrc-notmain
     89 TESTS+=		depsrc-optional
     90 TESTS+=		depsrc-phony
     91 TESTS+=		depsrc-precious
     92 TESTS+=		depsrc-recursive
     93 TESTS+=		depsrc-silent
     94 TESTS+=		depsrc-use
     95 TESTS+=		depsrc-usebefore
     96 TESTS+=		depsrc-usebefore-double-colon
     97 TESTS+=		depsrc-wait
     98 TESTS+=		deptgt
     99 TESTS+=		deptgt-begin
    100 TESTS+=		deptgt-default
    101 TESTS+=		deptgt-delete_on_error
    102 TESTS+=		deptgt-end
    103 TESTS+=		deptgt-error
    104 TESTS+=		deptgt-ignore
    105 TESTS+=		deptgt-interrupt
    106 TESTS+=		deptgt-main
    107 TESTS+=		deptgt-makeflags
    108 TESTS+=		deptgt-no_parallel
    109 TESTS+=		deptgt-nopath
    110 TESTS+=		deptgt-notparallel
    111 TESTS+=		deptgt-objdir
    112 TESTS+=		deptgt-order
    113 TESTS+=		deptgt-path
    114 TESTS+=		deptgt-path-suffix
    115 TESTS+=		deptgt-phony
    116 TESTS+=		deptgt-precious
    117 TESTS+=		deptgt-shell
    118 TESTS+=		deptgt-silent
    119 TESTS+=		deptgt-stale
    120 TESTS+=		deptgt-suffixes
    121 TESTS+=		dir
    122 TESTS+=		dir-expand-path
    123 TESTS+=		directive
    124 TESTS+=		directive-elif
    125 TESTS+=		directive-elifdef
    126 TESTS+=		directive-elifmake
    127 TESTS+=		directive-elifndef
    128 TESTS+=		directive-elifnmake
    129 TESTS+=		directive-else
    130 TESTS+=		directive-endif
    131 TESTS+=		directive-error
    132 TESTS+=		directive-export
    133 TESTS+=		directive-export-env
    134 TESTS+=		directive-export-literal
    135 TESTS+=		directive-for
    136 TESTS+=		directive-for-generating-endif
    137 TESTS+=		directive-if
    138 TESTS+=		directive-ifdef
    139 TESTS+=		directive-ifmake
    140 TESTS+=		directive-ifndef
    141 TESTS+=		directive-ifnmake
    142 TESTS+=		directive-info
    143 TESTS+=		directive-undef
    144 TESTS+=		directive-unexport
    145 TESTS+=		directive-unexport-env
    146 TESTS+=		directive-warning
    147 TESTS+=		directives
    148 TESTS+=		dollar
    149 TESTS+=		doterror
    150 TESTS+=		dotwait
    151 TESTS+=		envfirst
    152 TESTS+=		error
    153 TESTS+=		# escape	# broken by reverting POSIX changes
    154 TESTS+=		export
    155 TESTS+=		export-all
    156 TESTS+=		export-env
    157 TESTS+=		export-variants
    158 TESTS+=		forloop
    159 TESTS+=		forsubst
    160 TESTS+=		impsrc
    161 TESTS+=		include-main
    162 TESTS+=		lint
    163 TESTS+=		make-exported
    164 TESTS+=		misc
    165 TESTS+=		moderrs
    166 TESTS+=		modmatch
    167 TESTS+=		modmisc
    168 TESTS+=		modts
    169 TESTS+=		modword
    170 TESTS+=		opt
    171 TESTS+=		opt-backwards
    172 TESTS+=		opt-chdir
    173 TESTS+=		opt-debug
    174 TESTS+=		opt-debug-all
    175 TESTS+=		opt-debug-archive
    176 TESTS+=		opt-debug-curdir
    177 TESTS+=		opt-debug-cond
    178 TESTS+=		opt-debug-dir
    179 TESTS+=		opt-debug-errors
    180 TESTS+=		opt-debug-file
    181 TESTS+=		opt-debug-for
    182 TESTS+=		opt-debug-graph1
    183 TESTS+=		opt-debug-graph2
    184 TESTS+=		opt-debug-graph3
    185 TESTS+=		opt-debug-hash
    186 TESTS+=		opt-debug-jobs
    187 TESTS+=		opt-debug-lint
    188 TESTS+=		opt-debug-loud
    189 TESTS+=		opt-debug-meta
    190 TESTS+=		opt-debug-making
    191 TESTS+=		opt-debug-no-rm
    192 TESTS+=		opt-debug-parse
    193 TESTS+=		opt-debug-suff
    194 TESTS+=		opt-debug-targets
    195 TESTS+=		opt-debug-varraw
    196 TESTS+=		opt-debug-var
    197 TESTS+=		opt-debug-x-trace
    198 TESTS+=		opt-define
    199 TESTS+=		opt-env
    200 TESTS+=		opt-file
    201 TESTS+=		opt-ignore
    202 TESTS+=		opt-include-dir
    203 TESTS+=		opt-jobs
    204 TESTS+=		opt-jobs-internal
    205 TESTS+=		opt-keep-going
    206 TESTS+=		opt-m-include-dir
    207 TESTS+=		opt-no-action
    208 TESTS+=		opt-no-action-at-all
    209 TESTS+=		opt-query
    210 TESTS+=		opt-raw
    211 TESTS+=		opt-silent
    212 TESTS+=		opt-touch
    213 TESTS+=		opt-tracefile
    214 TESTS+=		opt-var-expanded
    215 TESTS+=		opt-var-literal
    216 TESTS+=		opt-warnings-as-errors
    217 TESTS+=		opt-where-am-i
    218 TESTS+=		opt-x-reduce-exported
    219 TESTS+=		order
    220 TESTS+=		phony-end
    221 TESTS+=		posix
    222 TESTS+=		# posix1	# broken by reverting POSIX changes
    223 TESTS+=		qequals
    224 TESTS+=		recursive
    225 TESTS+=		sh
    226 TESTS+=		sh-dots
    227 TESTS+=		sh-jobs
    228 TESTS+=		sh-jobs-error
    229 TESTS+=		sh-leading-at
    230 TESTS+=		sh-leading-hyphen
    231 TESTS+=		sh-leading-plus
    232 TESTS+=		sh-meta-chars
    233 TESTS+=		sh-multi-line
    234 TESTS+=		sh-single-line
    235 TESTS+=		# suffixes	# runs into an endless loop (try -dA)
    236 TESTS+=		sunshcmd
    237 TESTS+=		sysv
    238 TESTS+=		ternary
    239 TESTS+=		unexport
    240 TESTS+=		unexport-env
    241 TESTS+=		use-inference
    242 TESTS+=		var-class
    243 TESTS+=		var-class-cmdline
    244 TESTS+=		var-class-env
    245 TESTS+=		var-class-global
    246 TESTS+=		var-class-local
    247 TESTS+=		var-class-local-legacy
    248 TESTS+=		var-op
    249 TESTS+=		var-op-append
    250 TESTS+=		var-op-assign
    251 TESTS+=		var-op-default
    252 TESTS+=		var-op-expand
    253 TESTS+=		var-op-shell
    254 TESTS+=		varcmd
    255 TESTS+=		vardebug
    256 TESTS+=		varfind
    257 TESTS+=		varmisc
    258 TESTS+=		varmod
    259 TESTS+=		varmod-assign
    260 TESTS+=		varmod-defined
    261 TESTS+=		varmod-edge
    262 TESTS+=		varmod-exclam-shell
    263 TESTS+=		varmod-extension
    264 TESTS+=		varmod-gmtime
    265 TESTS+=		varmod-hash
    266 TESTS+=		varmod-head
    267 TESTS+=		varmod-ifelse
    268 TESTS+=		varmod-l-name-to-value
    269 TESTS+=		varmod-localtime
    270 TESTS+=		varmod-loop
    271 TESTS+=		varmod-match
    272 TESTS+=		varmod-match-escape
    273 TESTS+=		varmod-no-match
    274 TESTS+=		varmod-order
    275 TESTS+=		varmod-order-reverse
    276 TESTS+=		varmod-order-shuffle
    277 TESTS+=		varmod-path
    278 TESTS+=		varmod-quote
    279 TESTS+=		varmod-quote-dollar
    280 TESTS+=		varmod-range
    281 TESTS+=		varmod-remember
    282 TESTS+=		varmod-root
    283 TESTS+=		varmod-select-words
    284 TESTS+=		varmod-shell
    285 TESTS+=		varmod-subst
    286 TESTS+=		varmod-subst-regex
    287 TESTS+=		varmod-sysv
    288 TESTS+=		varmod-tail
    289 TESTS+=		varmod-to-abs
    290 TESTS+=		varmod-to-lower
    291 TESTS+=		varmod-to-many-words
    292 TESTS+=		varmod-to-one-word
    293 TESTS+=		varmod-to-separator
    294 TESTS+=		varmod-to-upper
    295 TESTS+=		varmod-undefined
    296 TESTS+=		varmod-unique
    297 TESTS+=		varname
    298 TESTS+=		varname-dollar
    299 TESTS+=		varname-dot-alltargets
    300 TESTS+=		varname-dot-curdir
    301 TESTS+=		varname-dot-includes
    302 TESTS+=		varname-dot-includedfromdir
    303 TESTS+=		varname-dot-includedfromfile
    304 TESTS+=		varname-dot-libs
    305 TESTS+=		varname-dot-make-dependfile
    306 TESTS+=		varname-dot-make-expand_variables
    307 TESTS+=		varname-dot-make-exported
    308 TESTS+=		varname-dot-make-jobs
    309 TESTS+=		varname-dot-make-jobs-prefix
    310 TESTS+=		varname-dot-make-level
    311 TESTS+=		varname-dot-make-makefile_preference
    312 TESTS+=		varname-dot-make-makefiles
    313 TESTS+=		varname-dot-make-meta-bailiwick
    314 TESTS+=		varname-dot-make-meta-created
    315 TESTS+=		varname-dot-make-meta-files
    316 TESTS+=		varname-dot-make-meta-ignore_filter
    317 TESTS+=		varname-dot-make-meta-ignore_paths
    318 TESTS+=		varname-dot-make-meta-ignore_patterns
    319 TESTS+=		varname-dot-make-meta-prefix
    320 TESTS+=		varname-dot-make-mode
    321 TESTS+=		varname-dot-make-path_filemon
    322 TESTS+=		varname-dot-make-pid
    323 TESTS+=		varname-dot-make-ppid
    324 TESTS+=		varname-dot-make-save_dollars
    325 TESTS+=		varname-dot-makeoverrides
    326 TESTS+=		varname-dot-newline
    327 TESTS+=		varname-dot-objdir
    328 TESTS+=		varname-dot-parsedir
    329 TESTS+=		varname-dot-parsefile
    330 TESTS+=		varname-dot-path
    331 TESTS+=		varname-dot-shell
    332 TESTS+=		varname-dot-targets
    333 TESTS+=		varname-empty
    334 TESTS+=		varname-make
    335 TESTS+=		varname-make_print_var_on_error
    336 TESTS+=		varname-makefile
    337 TESTS+=		varname-makeflags
    338 TESTS+=		varname-pwd
    339 TESTS+=		varname-vpath
    340 TESTS+=		varparse-dynamic
    341 TESTS+=		varquote
    342 TESTS+=		varshell
    343 
    344 # Additional environment variables for some of the tests.
    345 # The base environment is -i PATH="$PATH".
    346 ENV.envfirst=		FROM_ENV=value-from-env
    347 ENV.varmisc=		FROM_ENV=env
    348 ENV.varmisc+=		FROM_ENV_BEFORE=env
    349 ENV.varmisc+=		FROM_ENV_AFTER=env
    350 
    351 # Override make flags for some of the tests; default is -k.
    352 # If possible, write ".MAKEFLAGS: -dv" in the test .mk file instead of
    353 # settings FLAGS.test=-dv here, since that is closer to the test code.
    354 FLAGS.counter=		-dv
    355 FLAGS.directive-ifmake=	first second
    356 FLAGS.doterror=		# none
    357 FLAGS.envfirst=		-e
    358 FLAGS.export=		# none
    359 FLAGS.lint=		-dL -k
    360 FLAGS.opt-ignore=	-i
    361 FLAGS.opt-keep-going=	-k
    362 FLAGS.opt-no-action=	-n
    363 FLAGS.opt-query=	-q
    364 FLAGS.opt-var-expanded=	-v VAR -v VALUE
    365 FLAGS.opt-var-literal=	-V VAR -V VALUE
    366 FLAGS.opt-warnings-as-errors= -W
    367 FLAGS.order=		-j1
    368 FLAGS.recursive=	-dL
    369 FLAGS.sh-leading-plus=	-n
    370 FLAGS.vardebug=		-k -dv FROM_CMDLINE=
    371 FLAGS.varmod-match-escape= -dv
    372 FLAGS.varname-dot-shell= -dpv
    373 FLAGS.varname-empty=	-dv '$${:U}=cmdline-u' '=cmline-plain'
    374 
    375 # Some tests need extra postprocessing.
    376 SED_CMDS.opt-debug-graph1= \
    377 			-e 's,${.CURDIR},CURDIR,'
    378 SED_CMDS.opt-debug-graph1+= \
    379 			-e '/Global Variables:/,/Suffixes:/d'
    380 SED_CMDS.sh-dots=	-e 's,^.*\.\.\.:.*,<normalized: ...: not found>,'
    381 SED_CMDS.varmod-subst-regex+= \
    382 			-e 's,\(Regex compilation error:\).*,\1 (details omitted),'
    383 SED_CMDS.varmod-edge+=	-e 's, line [0-9]*:, line omitted:,'
    384 SED_CMDS.varshell+=	-e 's,^${.SHELL:T}: ,,'
    385 SED_CMDS.varshell+=	-e '/command/s,No such.*,not found,'
    386 SED_CMDS.varname-dot-shell=	-e 's, = /.*, = (details omitted),'
    387 SED_CMDS.varname-dot-shell+=	-e 's,"/[^"]*","(details omitted)",'
    388 SED_CMDS.varname-dot-shell+=	-e 's,\[/[^]]*\],[(details omitted)],'
    389 
    390 # Some tests need an additional round of postprocessing.
    391 POSTPROC.counter=	${TOOL_SED} -n -e '/:RELEVANT = yes/,/:RELEVANT = no/p'
    392 POSTPROC.deptgt-suffixes= \
    393 			${TOOL_SED} -n -e '/^\#\*\*\* Suffixes/,/^\#\*/p'
    394 POSTPROC.vardebug=	${TOOL_SED} -n -e '/:RELEVANT = yes/,/:RELEVANT = no/p'
    395 POSTPROC.varmod-match-escape= ${TOOL_SED} -n -e '/^Pattern/p'
    396 POSTPROC.varname=	${TOOL_SED} -n -e '/^MAGIC/p' -e '/^ORDER_/p'
    397 POSTPROC.varname-dot-shell= \
    398 			awk '/\.SHELL/ || /^ParseReadLine/'
    399 POSTPROC.varname-empty=	${TOOL_SED} -n -e '/^Var_Set/p' -e '/^out:/p'
    400 
    401 # Some tests reuse other tests, which makes them unnecessarily fragile.
    402 export-all.rawout: export.mk
    403 unexport.rawout: export.mk
    404 unexport-env.rawout: export.mk
    405 
    406 # End of the configuration section.
    407 
    408 .MAIN: all
    409 
    410 UNIT_TESTS:=	${.PARSEDIR}
    411 .PATH: ${UNIT_TESTS}
    412 
    413 OUTFILES=	${TESTS:=.out}
    414 
    415 all: ${OUTFILES}
    416 
    417 CLEANFILES=		*.rawout *.out *.status *.tmp *.core *.tmp
    418 CLEANFILES+=		obj*.[och] lib*.a	# posix1.mk
    419 CLEANFILES+=		issue* .[ab]*		# suffixes.mk
    420 CLEANDIRS=		dir dummy		# posix1.mk
    421 
    422 clean:
    423 	rm -f ${CLEANFILES}
    424 	rm -rf ${CLEANDIRS}
    425 
    426 TEST_MAKE?=	${.MAKE}
    427 TOOL_SED?=	sed
    428 
    429 # ensure consistent results from sort(1)
    430 LC_ALL=		C
    431 LANG=		C
    432 .export LANG LC_ALL
    433 
    434 # Each test is run in a sub-make, to keep the tests for interfering with
    435 # each other, and because they use different environment variables and
    436 # command line options.
    437 .SUFFIXES: .mk .rawout .out
    438 .mk.rawout:
    439 	@${_MKMSG_TEST:Uecho '#      test '} ${.PREFIX}
    440 	@set -eu; \
    441 	cd ${.OBJDIR}; \
    442 	env -i PATH="$$PATH" ${ENV.${.TARGET:R}} \
    443 	  ${TEST_MAKE} \
    444 	    -r -C ${.CURDIR} -f ${.IMPSRC} \
    445 	    ${FLAGS.${.TARGET:R}:U-k} \
    446 	    > ${.TARGET}.tmp 2>&1 \
    447 	&& status=$$? || status=$$?; \
    448 	echo $$status > ${.TARGET:R}.status
    449 	@mv ${.TARGET}.tmp ${.TARGET}
    450 
    451 # Postprocess the test output so that the results can be compared.
    452 #
    453 # always pretend .MAKE was called 'make'
    454 _SED_CMDS+=	-e 's,^${TEST_MAKE:T:S,.,\\.,g}[][0-9]*:,make:,'
    455 _SED_CMDS+=	-e 's,${TEST_MAKE:S,.,\\.,g},make,'
    456 # replace anything after 'stopped in' with unit-tests
    457 _SED_CMDS+=	-e '/stopped/s, /.*, unit-tests,'
    458 # strip ${.CURDIR}/ from the output
    459 _SED_CMDS+=	-e 's,${.CURDIR:S,.,\\.,g}/,,g'
    460 _SED_CMDS+=	-e 's,${UNIT_TESTS:S,.,\\.,g}/,,g'
    461 
    462 .rawout.out:
    463 	@${TOOL_SED} ${_SED_CMDS} ${SED_CMDS.${.TARGET:R}} \
    464 	  < ${.IMPSRC} > ${.TARGET}.tmp1
    465 	@${POSTPROC.${.TARGET:R}:Ucat} < ${.TARGET}.tmp1 > ${.TARGET}.tmp2
    466 	@rm ${.TARGET}.tmp1
    467 	@echo "exit status `cat ${.TARGET:R}.status`" >> ${.TARGET}.tmp2
    468 	@mv ${.TARGET}.tmp2 ${.TARGET}
    469 
    470 # Compare all output files
    471 test:	${OUTFILES} .PHONY
    472 	@failed= ; \
    473 	for test in ${TESTS}; do \
    474 	  diff -u ${UNIT_TESTS}/$${test}.exp $${test}.out \
    475 	  || failed="$${failed}$${failed:+ }$${test}" ; \
    476 	done ; \
    477 	if [ -n "$${failed}" ]; then \
    478 	  echo "Failed tests: $${failed}" ; false ; \
    479 	else \
    480 	  echo "All tests passed" ; \
    481 	fi
    482 
    483 accept:
    484 	@for test in ${TESTS}; do \
    485 	  cmp -s ${UNIT_TESTS}/$${test}.exp $${test}.out \
    486 	  || { echo "Replacing $${test}.exp" ; \
    487 	       cp $${test}.out ${UNIT_TESTS}/$${test}.exp ; } \
    488 	done
    489 
    490 # Note: only works for adding tests.
    491 # To remove a test, the $$mi file must be edited manually.
    492 sync-mi:
    493 	@set -eu;							\
    494 	cd "${MAKEFILE:tA:H}/../../..";					\
    495 	mi="distrib/sets/lists/tests/mi";				\
    496 	cvs update "$$mi";						\
    497 	testsdir="usr.bin/make/unit-tests";				\
    498 	fmt="./usr/tests/$$testsdir/%s\ttests-usr.bin-tests\tcompattestfile,atf\\n"; \
    499 	(cd "$$testsdir" && ls *.exp *.mk) | xargs printf "$$fmt" >> "$$mi"; \
    500 	distrib/sets/fmt-list "$$mi";					\
    501 	cvs diff "$$mi" || true
    502 
    503 .if exists(${TEST_MAKE})
    504 ${TESTS:=.rawout}: ${TEST_MAKE} ${.PARSEDIR}/Makefile
    505 .endif
    506 
    507 .-include <bsd.obj.mk>
    508