Makefile revision 1.92 
1# $NetBSD: Makefile,v 1.92 2020/08/15 01:49:07 rillig Exp $
2#
3# Unit tests for make(1)
4#
5# The main targets are:
6#
7# all:
8#	run all the tests
9# test:
10#	run 'all', and compare to expected results
11# accept:
12#	move generated output to expected results
13#
14# Settable variables
15#
16# TEST_MAKE
17#	The make program to be tested.
18#
19#
20# Adding a test case
21#
22# Each feature should get its own set of tests in its own suitably
23# named makefile (*.mk), with its own set of expected results (*.exp),
24# and it should be added to the TESTS list.
25#
26# Any added files must also be added to src/distrib/sets/lists/tests/mi.
27# To do that, just run "make sync-mi" in this directory.
28#
29# A few *.mk files are helper files for other tests (such as include-sub.mk)
30# and are thus not added to TESTS.  Such files must be ignored in
31# src/tests/usr.bin/make/t_make.sh.
32#
33
34# Each test is in a sub-makefile.
35# Keep the list sorted.
36# Any test that is commented out must be ignored in
37# src/tests/usr.bin/make/t_make.sh as well.
38TESTS+=		# archive	# broken on FreeBSD, enabled in t_make.sh
39TESTS+=		cmdline
40TESTS+=		comment
41TESTS+=		cond-late
42TESTS+=		cond-short
43TESTS+=		cond1
44TESTS+=		cond2
45TESTS+=		counter
46TESTS+=		dir
47TESTS+=		directives
48TESTS+=		dollar
49TESTS+=		doterror
50TESTS+=		dotwait
51TESTS+=		envfirst
52TESTS+=		error
53TESTS+=		# escape	# broken by reverting POSIX changes
54TESTS+=		export
55TESTS+=		export-all
56TESTS+=		export-env
57TESTS+=		export-variants
58TESTS+=		forloop
59TESTS+=		forsubst
60TESTS+=		hash
61TESTS+=		impsrc
62TESTS+=		include-main
63TESTS+=		lint
64TESTS+=		make-exported
65TESTS+=		misc
66TESTS+=		moderrs
67TESTS+=		modmatch
68TESTS+=		modmisc
69TESTS+=		modorder
70TESTS+=		modts
71TESTS+=		modword
72TESTS+=		order
73TESTS+=		phony-end
74TESTS+=		posix
75TESTS+=		# posix1	# broken by reverting POSIX changes
76TESTS+=		qequals
77TESTS+=		recursive
78TESTS+=		# suffixes	# runs into an endless loop (try -dA)
79TESTS+=		sunshcmd
80TESTS+=		sysv
81TESTS+=		ternary
82TESTS+=		unexport
83TESTS+=		unexport-env
84TESTS+=		use-inference
85TESTS+=		varcmd
86TESTS+=		vardebug
87TESTS+=		varfind
88TESTS+=		varmisc
89TESTS+=		varmod-edge
90TESTS+=		varparse-dynamic
91TESTS+=		varquote
92TESTS+=		varshell
93
94# Override environment variables for some of the tests.
95ENV.counter=		-i
96ENV.envfirst=		FROM_ENV=value-from-env
97ENV.export=		-i PATH=${PATH:Q}
98ENV.export-variants=	-i PATH=${PATH:Q}
99ENV.lint=		-i
100ENV.make-exported=	-i PATH=${PATH:Q}
101ENV.recursive=		-i
102ENV.varmisc=		FROM_ENV=env
103ENV.varmisc+=		FROM_ENV_BEFORE=env
104ENV.varmisc+=		FROM_ENV_AFTER=env
105
106# Override make flags for some of the tests; default is -k.
107FLAGS.counter=		-r -dv
108FLAGS.doterror=		# none
109FLAGS.envfirst=		-e
110FLAGS.export=		-r
111FLAGS.lint=		-dL -k
112FLAGS.order=		-j1
113FLAGS.recursive=	-dL -r
114FLAGS.vardebug=		-k -dv FROM_CMDLINE=
115
116# Some tests need extra post-processing.
117SED_CMDS.moderrs+=	-e 's,\(Regex compilation error:\).*,\1 (details omitted),'
118SED_CMDS.modmisc+=	-e 's,\(Regex compilation error:\).*,\1 (details omitted),'
119SED_CMDS.varmod-edge+=	-e 's, line [0-9]*:, line omitted:,'
120SED_CMDS.varshell+=	-e 's,^[a-z]*sh: ,,'
121SED_CMDS.varshell+=	-e '/command/s,No such.*,not found,'
122
123# Some tests need an additional round of postprocessing.
124POSTPROC.counter=	${TOOL_SED} -n -e '/:RELEVANT = yes/,/:RELEVANT = no/p'
125POSTPROC.vardebug=	${TOOL_SED} -n -e '/:RELEVANT = yes/,/:RELEVANT = no/p'
126
127export-all.rawout: export.mk
128unexport.rawout: export.mk
129unexport-env.rawout: export.mk
130
131# End of the configuration section.
132
133.MAIN: all
134
135UNIT_TESTS:=	${.PARSEDIR}
136.PATH: ${UNIT_TESTS}
137
138OUTFILES=	${TESTS:=.out}
139
140all: ${OUTFILES}
141
142CLEANFILES+=		*.rawout *.out *.status *.tmp *.core *.tmp
143CLEANFILES+=		obj*.[och] lib*.a	# posix1.mk
144CLEANFILES+=		issue* .[ab]*		# suffixes.mk
145CLEANRECURSIVE+=	dir dummy		# posix1.mk
146
147clean:
148	rm -f ${CLEANFILES}
149.if !empty(CLEANRECURSIVE)
150	rm -rf ${CLEANRECURSIVE}
151.endif
152
153TEST_MAKE?=	${.MAKE}
154TOOL_SED?=	sed
155
156# ensure consistent results from sort(1)
157LC_ALL=		C
158LANG=		C
159.export LANG LC_ALL
160
161# the tests are actually done with sub-makes.
162.SUFFIXES: .mk .rawout .out
163.mk.rawout:
164	@echo testing ${.IMPSRC}
165	@set -eu; \
166	cd ${.OBJDIR}; \
167	env ${ENV.${.TARGET:R}} ${TEST_MAKE} -C ${.CURDIR} \
168	  ${FLAGS.${.TARGET:R}:U-k} -f ${.IMPSRC} \
169	  > ${.TARGET}.tmp 2>&1 \
170	&& status=$$? || status=$$?; \
171	echo $$status > ${.TARGET:R}.status
172	@mv ${.TARGET}.tmp ${.TARGET}
173
174# Post-process the test output so that the results can be compared.
175#
176# always pretend .MAKE was called 'make'
177_SED_CMDS+=	-e 's,^${TEST_MAKE:T:S,.,\\.,g}[][0-9]*:,make:,'
178_SED_CMDS+=	-e 's,${TEST_MAKE:S,.,\\.,g},make,'
179# replace anything after 'stopped in' with unit-tests
180_SED_CMDS+=	-e '/stopped/s, /.*, unit-tests,'
181# strip ${.CURDIR}/ from the output
182_SED_CMDS+=	-e 's,${.CURDIR:S,.,\\.,g}/,,g'
183_SED_CMDS+=	-e 's,${UNIT_TESTS:S,.,\\.,g}/,,g'
184
185.rawout.out:
186	@echo postprocess ${.TARGET}
187	@${TOOL_SED} ${_SED_CMDS} ${SED_CMDS.${.TARGET:R}} \
188	  < ${.IMPSRC} > ${.TARGET}.tmp1
189	@${POSTPROC.${.TARGET:R}:Ucat} < ${.TARGET}.tmp1 > ${.TARGET}.tmp2
190	@rm ${.TARGET}.tmp1
191	@echo "exit status `cat ${.TARGET:R}.status`" >> ${.TARGET}.tmp2
192	@mv ${.TARGET}.tmp2 ${.TARGET}
193
194# Compare all output files
195test:	${OUTFILES} .PHONY
196	@failed= ; \
197	for test in ${TESTS}; do \
198	  diff -u ${UNIT_TESTS}/$${test}.exp $${test}.out \
199	  || failed="$${failed}$${failed:+ }$${test}" ; \
200	done ; \
201	if [ -n "$${failed}" ]; then \
202	  echo "Failed tests: $${failed}" ; false ; \
203	else \
204	  echo "All tests passed" ; \
205	fi
206
207accept:
208	@for test in ${TESTS}; do \
209	  cmp -s ${UNIT_TESTS}/$${test}.exp $${test}.out \
210	  || { echo "Replacing $${test}.exp" ; \
211	       cp $${test}.out ${UNIT_TESTS}/$${test}.exp ; } \
212	done
213
214SYNC_MI_AWK= \
215	BEGIN {								\
216	  testsdir = "usr.bin/make/unit-tests";				\
217	  linestart = "./usr/tests/" testsdir;				\
218	  fmt = linestart "/%s\ttests-usr.bin-tests\tcompattestfile,atf\\n"; \
219	  cmd = "cd " testsdir " && ls *.exp *.mk | xargs printf '" fmt "'" \
220	}								\
221	function startswith(s, prefix) {				\
222	  return substr(s, 1, length(prefix)) == prefix;		\
223	}								\
224	startswith($$1, linestart) && $$1 ~ /\.(exp|mk)$$/ { next }	\
225	{ print $$0 }							\
226	$$1 == linestart "/Makefile" { system(cmd) }
227
228sync-mi:
229	@set -eu;							\
230	cd "${MAKEFILE:tA:H}/../../..";					\
231	mi="distrib/sets/lists/tests/mi";				\
232	awk ${SYNC_MI_AWK:Q} < "$$mi" > "$$mi.$@";			\
233	mv -f "$$mi.$@" "$$mi";						\
234	cvs diff "$$mi" || true
235
236.if exists(${TEST_MAKE})
237${TESTS:=.rawout}: ${TEST_MAKE}
238.endif
239
240.-include <bsd.obj.mk>
241