Home | History | Annotate | Line # | Download | only in unit-tests
Makefile revision 1.79 
      1 # $NetBSD: Makefile,v 1.79 2020/08/01 15:28:28 rillig Exp $
      2 #
      3 # Unit tests for make(1)
      4 #
      5 # The main targets are:
      6 #
      7 # all:
      8 #	run all the tests
      9 # test:
     10 #	run 'all', and compare to expected results
     11 # accept:
     12 #	move generated output to expected results
     13 #
     14 # Settable variables
     15 #
     16 # TEST_MAKE
     17 #	The make program to be tested.
     18 #
     19 #
     20 # Adding a test case
     21 #
     22 # Each feature should get its own set of tests in its own suitably
     23 # named makefile (*.mk), with its own set of expected results (*.exp),
     24 # and it should be added to the TESTS list.
     25 #
     26 # Any added files must also be added to src/distrib/sets/lists/tests/mi.
     27 # To do that, just run "make sync-mi" in this directory.
     28 #
     29 # A few *.mk files are helper files for other tests (such as include-sub.mk)
     30 # and are thus not added to TESTS.  Such files must be ignored in
     31 # src/tests/usr.bin/make/t_make.sh.
     32 #
     33 
     34 # Each test is in a sub-makefile.
     35 # Keep the list sorted.
     36 TESTS+=		# archive	# broken on FreeBSD
     37 TESTS+=		cmdline
     38 TESTS+=		comment
     39 TESTS+=		cond-late
     40 TESTS+=		cond-short
     41 TESTS+=		cond1
     42 TESTS+=		cond2
     43 TESTS+=		dir
     44 TESTS+=		directives
     45 TESTS+=		dollar
     46 TESTS+=		doterror
     47 TESTS+=		dotwait
     48 TESTS+=		envfirst
     49 TESTS+=		error
     50 TESTS+=		# escape	# broken by reverting POSIX changes
     51 TESTS+=		export
     52 TESTS+=		export-all
     53 TESTS+=		export-env
     54 TESTS+=		forloop
     55 TESTS+=		forsubst
     56 TESTS+=		hash
     57 TESTS+=		# impsrc	# broken by reverting POSIX changes
     58 TESTS+=		include-main
     59 TESTS+=		misc
     60 TESTS+=		moderrs
     61 TESTS+=		modmatch
     62 TESTS+=		modmisc
     63 TESTS+=		modorder
     64 TESTS+=		modts
     65 TESTS+=		modword
     66 TESTS+=		order
     67 TESTS+=		# phony-end	# broken by reverting POSIX changes
     68 TESTS+=		posix
     69 TESTS+=		# posix1	# broken by reverting POSIX changes
     70 TESTS+=		qequals
     71 TESTS+=		# suffixes	# broken by reverting POSIX changes
     72 TESTS+=		sunshcmd
     73 TESTS+=		sysv
     74 TESTS+=		ternary
     75 TESTS+=		unexport
     76 TESTS+=		unexport-env
     77 TESTS+=		varcmd
     78 TESTS+=		vardebug
     79 TESTS+=		varfind
     80 TESTS+=		varmisc
     81 TESTS+=		varmod-edge
     82 TESTS+=		varparse-dynamic
     83 TESTS+=		varquote
     84 TESTS+=		varshell
     85 
     86 # Override environment variables for some of the tests.
     87 ENV.envfirst=		FROM_ENV=value-from-env
     88 ENV.export=		-i PATH=${PATH:Q}
     89 ENV.varmisc=		FROM_ENV=env
     90 ENV.varmisc+=		FROM_ENV_BEFORE=env
     91 ENV.varmisc+=		FROM_ENV_AFTER=env
     92 
     93 # Override make flags for some of the tests; default is -k.
     94 FLAGS.doterror=		# none
     95 FLAGS.envfirst=		-e
     96 FLAGS.export=		-r
     97 FLAGS.order=		-j1
     98 FLAGS.vardebug=		-k -dv FROM_CMDLINE=
     99 
    100 # Some tests need extra post-processing.
    101 SED_CMDS.moderrs+=	-e 's,\(substitution error:\).*,\1 (details omitted),'
    102 SED_CMDS.modmisc+=	-e 's,\(substitution error:\).*,\1 (details omitted),'
    103 SED_CMDS.varmod-edge+=	-e 's, line [0-9]*:, line omitted:,'
    104 SED_CMDS.varshell+=	-e 's,^[a-z]*sh: ,,'
    105 SED_CMDS.varshell+=	-e '/command/s,No such.*,not found,'
    106 
    107 # Some tests need an additional round of postprocessing.
    108 POSTPROC.vardebug=	${TOOL_SED} -n -e '/:RELEVANT = yes/,/:RELEVANT = no/p'
    109 
    110 # End of the configuration section.
    111 
    112 .MAIN: all
    113 
    114 UNIT_TESTS:=	${.PARSEDIR}
    115 .PATH: ${UNIT_TESTS}
    116 
    117 OUTFILES=	${TESTS:=.out}
    118 
    119 all: ${OUTFILES}
    120 
    121 CLEANFILES+=		*.rawout *.out *.status *.tmp *.core *.tmp
    122 CLEANFILES+=		obj*.[och] lib*.a	# posix1.mk
    123 CLEANFILES+=		issue* .[ab]*		# suffixes.mk
    124 CLEANRECURSIVE+=	dir dummy		# posix1.mk
    125 
    126 clean:
    127 	rm -f ${CLEANFILES}
    128 .if !empty(CLEANRECURSIVE)
    129 	rm -rf ${CLEANRECURSIVE}
    130 .endif
    131 
    132 TEST_MAKE?=	${.MAKE}
    133 TOOL_SED?=	sed
    134 
    135 # ensure consistent results from sort(1)
    136 LC_ALL=		C
    137 LANG=		C
    138 .export LANG LC_ALL
    139 
    140 # the tests are actually done with sub-makes.
    141 .SUFFIXES: .mk .rawout .out
    142 .mk.rawout:
    143 	@echo testing ${.IMPSRC}
    144 	@set -eu; \
    145 	cd ${.OBJDIR}; \
    146 	env ${ENV.${.TARGET:R}} ${TEST_MAKE} -C ${.CURDIR} \
    147 	  ${FLAGS.${.TARGET:R}:U-k} -f ${.IMPSRC} \
    148 	  > ${.TARGET}.tmp 2>&1 \
    149 	&& status=$$? || status=$$?; \
    150 	echo $$status > ${.TARGET:R}.status
    151 	@mv ${.TARGET}.tmp ${.TARGET}
    152 
    153 # Post-process the test output so that the results can be compared.
    154 #
    155 # always pretend .MAKE was called 'make'
    156 _SED_CMDS+=	-e 's,^${TEST_MAKE:T:S,.,\\.,g}[][0-9]*:,make:,'
    157 _SED_CMDS+=	-e 's,${TEST_MAKE:S,.,\\.,g},make,'
    158 # replace anything after 'stopped in' with unit-tests
    159 _SED_CMDS+=	-e '/stopped/s, /.*, unit-tests,'
    160 # strip ${.CURDIR}/ from the output
    161 _SED_CMDS+=	-e 's,${.CURDIR:S,.,\\.,g}/,,g'
    162 _SED_CMDS+=	-e 's,${UNIT_TESTS:S,.,\\.,g}/,,g'
    163 
    164 .rawout.out:
    165 	@echo postprocess ${.TARGET}
    166 	@${TOOL_SED} ${_SED_CMDS} ${SED_CMDS.${.TARGET:R}} \
    167 	  < ${.IMPSRC} > ${.TARGET}.tmp1
    168 	@${POSTPROC.${.TARGET:R}:Ucat} < ${.TARGET}.tmp1 > ${.TARGET}.tmp2
    169 	@rm ${.TARGET}.tmp1
    170 	@echo "exit status `cat ${.TARGET:R}.status`" >> ${.TARGET}.tmp2
    171 	@mv ${.TARGET}.tmp2 ${.TARGET}
    172 
    173 # Compare all output files
    174 test:	${OUTFILES} .PHONY
    175 	@failed= ; \
    176 	for test in ${TESTS}; do \
    177 	  diff -u ${UNIT_TESTS}/$${test}.exp $${test}.out \
    178 	  || failed="$${failed}$${failed:+ }$${test}" ; \
    179 	done ; \
    180 	if [ -n "$${failed}" ]; then \
    181 	  echo "Failed tests: $${failed}" ; false ; \
    182 	else \
    183 	  echo "All tests passed" ; \
    184 	fi
    185 
    186 accept:
    187 	@for test in ${TESTS}; do \
    188 	  cmp -s ${UNIT_TESTS}/$${test}.exp $${test}.out \
    189 	  || { echo "Replacing $${test}.exp" ; \
    190 	       cp $${test}.out ${UNIT_TESTS}/$${test}.exp ; } \
    191 	done
    192 
    193 SYNC_MI_AWK= \
    194 	BEGIN {								\
    195 	  testsdir = "usr.bin/make/unit-tests";				\
    196 	  linestart = "./usr/tests/" testsdir;				\
    197 	  fmt = linestart "/%s\ttests-usr.bin-tests\tcompattestfile,atf\\n"; \
    198 	  cmd = "cd " testsdir " && ls *.exp *.mk | xargs printf '" fmt "'" \
    199 	}								\
    200 	function startswith(s, prefix) {				\
    201 	  return substr(s, 1, length(prefix)) == prefix;		\
    202 	}								\
    203 	startswith($$1, linestart) && $$1 ~ /\.(exp|mk)$$/ { next }	\
    204 	{ print $$0 }							\
    205 	$$1 == linestart "/Makefile" { system(cmd) }
    206 
    207 sync-mi:
    208 	@set -eu;							\
    209 	cd "${MAKEFILE:tA:H}/../../..";					\
    210 	mi="distrib/sets/lists/tests/mi";				\
    211 	awk ${SYNC_MI_AWK:Q} < "$$mi" > "$$mi.$@";			\
    212 	mv -f "$$mi.$@" "$$mi";						\
    213 	cvs diff "$$mi" || true
    214 
    215 .if exists(${TEST_MAKE})
    216 ${TESTS:=.rawout}: ${TEST_MAKE}
    217 .endif
    218 
    219 .-include <bsd.obj.mk>
    220