Makefile revision 1.139 1 # $NetBSD: Makefile,v 1.139 2020/09/13 09:17:47 rillig Exp $
2 #
3 # Unit tests for make(1)
4 #
5 # The main targets are:
6 #
7 # all:
8 # run all the tests
9 # test:
10 # run 'all', and compare to expected results
11 # accept:
12 # move generated output to expected results
13 #
14 # Settable variables
15 #
16 # TEST_MAKE
17 # The make program to be tested.
18 #
19 #
20 # Adding a test case
21 #
22 # Each feature should get its own set of tests in its own suitably
23 # named makefile (*.mk), with its own set of expected results (*.exp),
24 # and it should be added to the TESTS list.
25 #
26 # Any added files must also be added to src/distrib/sets/lists/tests/mi.
27 # To do that, just run "make sync-mi" in this directory.
28 #
29 # A few *.mk files are helper files for other tests (such as include-sub.mk)
30 # and are thus not added to TESTS. Such files must be ignored in
31 # src/tests/usr.bin/make/t_make.sh.
32 #
33
34 # Each test is in a sub-makefile.
35 # Keep the list sorted.
36 # Any test that is commented out must be ignored in
37 # src/tests/usr.bin/make/t_make.sh as well.
38 TESTS+= archive
39 TESTS+= archive-suffix
40 TESTS+= cmd-interrupt
41 TESTS+= cmdline
42 TESTS+= comment
43 TESTS+= cond-cmp-numeric
44 TESTS+= cond-cmp-numeric-eq
45 TESTS+= cond-cmp-numeric-ge
46 TESTS+= cond-cmp-numeric-gt
47 TESTS+= cond-cmp-numeric-le
48 TESTS+= cond-cmp-numeric-lt
49 TESTS+= cond-cmp-numeric-ne
50 TESTS+= cond-cmp-string
51 TESTS+= cond-func
52 TESTS+= cond-func-commands
53 TESTS+= cond-func-defined
54 TESTS+= cond-func-empty
55 TESTS+= cond-func-exists
56 TESTS+= cond-func-make
57 TESTS+= cond-func-target
58 TESTS+= cond-late
59 TESTS+= cond-op
60 TESTS+= cond-op-and
61 TESTS+= cond-op-not
62 TESTS+= cond-op-or
63 TESTS+= cond-op-parentheses
64 TESTS+= cond-short
65 TESTS+= cond-token-number
66 TESTS+= cond-token-plain
67 TESTS+= cond-token-string
68 TESTS+= cond-token-var
69 TESTS+= cond1
70 TESTS+= cond2
71 TESTS+= counter
72 TESTS+= dep
73 TESTS+= dep-colon
74 TESTS+= dep-double-colon
75 TESTS+= dep-exclam
76 TESTS+= dep-none
77 TESTS+= dep-var
78 TESTS+= dep-wildcards
79 TESTS+= depsrc
80 TESTS+= depsrc-exec
81 TESTS+= depsrc-ignore
82 TESTS+= depsrc-made
83 TESTS+= depsrc-make
84 TESTS+= depsrc-meta
85 TESTS+= depsrc-nometa
86 TESTS+= depsrc-nometa_cmp
87 TESTS+= depsrc-nopath
88 TESTS+= depsrc-notmain
89 TESTS+= depsrc-optional
90 TESTS+= depsrc-phony
91 TESTS+= depsrc-precious
92 TESTS+= depsrc-recursive
93 TESTS+= depsrc-silent
94 TESTS+= depsrc-use
95 TESTS+= depsrc-usebefore
96 TESTS+= depsrc-usebefore-double-colon
97 TESTS+= depsrc-wait
98 TESTS+= deptgt
99 TESTS+= deptgt-begin
100 TESTS+= deptgt-default
101 TESTS+= deptgt-delete_on_error
102 TESTS+= deptgt-end
103 TESTS+= deptgt-error
104 TESTS+= deptgt-ignore
105 TESTS+= deptgt-interrupt
106 TESTS+= deptgt-main
107 TESTS+= deptgt-makeflags
108 TESTS+= deptgt-no_parallel
109 TESTS+= deptgt-nopath
110 TESTS+= deptgt-notparallel
111 TESTS+= deptgt-objdir
112 TESTS+= deptgt-order
113 TESTS+= deptgt-path
114 TESTS+= deptgt-path-suffix
115 TESTS+= deptgt-phony
116 TESTS+= deptgt-precious
117 TESTS+= deptgt-shell
118 TESTS+= deptgt-silent
119 TESTS+= deptgt-stale
120 TESTS+= deptgt-suffixes
121 TESTS+= dir
122 TESTS+= dir-expand-path
123 TESTS+= directive
124 TESTS+= directive-elif
125 TESTS+= directive-elifdef
126 TESTS+= directive-elifmake
127 TESTS+= directive-elifndef
128 TESTS+= directive-elifnmake
129 TESTS+= directive-else
130 TESTS+= directive-endif
131 TESTS+= directive-error
132 TESTS+= directive-export
133 TESTS+= directive-export-env
134 TESTS+= directive-export-literal
135 TESTS+= directive-for
136 TESTS+= directive-for-generating-endif
137 TESTS+= directive-if
138 TESTS+= directive-ifdef
139 TESTS+= directive-ifmake
140 TESTS+= directive-ifndef
141 TESTS+= directive-ifnmake
142 TESTS+= directive-info
143 TESTS+= directive-undef
144 TESTS+= directive-unexport
145 TESTS+= directive-unexport-env
146 TESTS+= directive-warning
147 TESTS+= directives
148 TESTS+= dollar
149 TESTS+= doterror
150 TESTS+= dotwait
151 TESTS+= envfirst
152 TESTS+= error
153 TESTS+= # escape # broken by reverting POSIX changes
154 TESTS+= export
155 TESTS+= export-all
156 TESTS+= export-env
157 TESTS+= export-variants
158 TESTS+= forloop
159 TESTS+= forsubst
160 TESTS+= impsrc
161 TESTS+= include-main
162 TESTS+= lint
163 TESTS+= make-exported
164 TESTS+= misc
165 TESTS+= moderrs
166 TESTS+= modmatch
167 TESTS+= modmisc
168 TESTS+= modts
169 TESTS+= modword
170 TESTS+= opt
171 TESTS+= opt-backwards
172 TESTS+= opt-chdir
173 TESTS+= opt-debug
174 TESTS+= opt-debug-all
175 TESTS+= opt-debug-archive
176 TESTS+= opt-debug-curdir
177 TESTS+= opt-debug-cond
178 TESTS+= opt-debug-dir
179 TESTS+= opt-debug-errors
180 TESTS+= opt-debug-file
181 TESTS+= opt-debug-for
182 TESTS+= opt-debug-graph1
183 TESTS+= opt-debug-graph2
184 TESTS+= opt-debug-graph3
185 TESTS+= opt-debug-hash
186 TESTS+= opt-debug-jobs
187 TESTS+= opt-debug-lint
188 TESTS+= opt-debug-loud
189 TESTS+= opt-debug-meta
190 TESTS+= opt-debug-making
191 TESTS+= opt-debug-no-rm
192 TESTS+= opt-debug-parse
193 TESTS+= opt-debug-suff
194 TESTS+= opt-debug-targets
195 TESTS+= opt-debug-varraw
196 TESTS+= opt-debug-var
197 TESTS+= opt-debug-x-trace
198 TESTS+= opt-define
199 TESTS+= opt-env
200 TESTS+= opt-file
201 TESTS+= opt-ignore
202 TESTS+= opt-include-dir
203 TESTS+= opt-jobs
204 TESTS+= opt-jobs-internal
205 TESTS+= opt-keep-going
206 TESTS+= opt-m-include-dir
207 TESTS+= opt-no-action
208 TESTS+= opt-no-action-at-all
209 TESTS+= opt-query
210 TESTS+= opt-raw
211 TESTS+= opt-silent
212 TESTS+= opt-touch
213 TESTS+= opt-tracefile
214 TESTS+= opt-var-expanded
215 TESTS+= opt-var-literal
216 TESTS+= opt-warnings-as-errors
217 TESTS+= opt-where-am-i
218 TESTS+= opt-x-reduce-exported
219 TESTS+= order
220 TESTS+= phony-end
221 TESTS+= posix
222 TESTS+= # posix1 # broken by reverting POSIX changes
223 TESTS+= qequals
224 TESTS+= recursive
225 TESTS+= sh
226 TESTS+= sh-dots
227 TESTS+= sh-jobs
228 TESTS+= sh-jobs-error
229 TESTS+= sh-leading-at
230 TESTS+= sh-leading-hyphen
231 TESTS+= sh-leading-plus
232 TESTS+= sh-meta-chars
233 TESTS+= sh-multi-line
234 TESTS+= sh-single-line
235 TESTS+= # suffixes # runs into an endless loop (try -dA)
236 TESTS+= sunshcmd
237 TESTS+= sysv
238 TESTS+= ternary
239 TESTS+= unexport
240 TESTS+= unexport-env
241 TESTS+= use-inference
242 TESTS+= var-class
243 TESTS+= var-class-cmdline
244 TESTS+= var-class-env
245 TESTS+= var-class-global
246 TESTS+= var-class-local
247 TESTS+= var-class-local-legacy
248 TESTS+= var-op
249 TESTS+= var-op-append
250 TESTS+= var-op-assign
251 TESTS+= var-op-default
252 TESTS+= var-op-expand
253 TESTS+= var-op-shell
254 TESTS+= varcmd
255 TESTS+= vardebug
256 TESTS+= varfind
257 TESTS+= varmisc
258 TESTS+= varmod
259 TESTS+= varmod-assign
260 TESTS+= varmod-defined
261 TESTS+= varmod-edge
262 TESTS+= varmod-exclam-shell
263 TESTS+= varmod-extension
264 TESTS+= varmod-gmtime
265 TESTS+= varmod-hash
266 TESTS+= varmod-head
267 TESTS+= varmod-ifelse
268 TESTS+= varmod-l-name-to-value
269 TESTS+= varmod-localtime
270 TESTS+= varmod-loop
271 TESTS+= varmod-match
272 TESTS+= varmod-match-escape
273 TESTS+= varmod-no-match
274 TESTS+= varmod-order
275 TESTS+= varmod-order-reverse
276 TESTS+= varmod-order-shuffle
277 TESTS+= varmod-path
278 TESTS+= varmod-quote
279 TESTS+= varmod-quote-dollar
280 TESTS+= varmod-range
281 TESTS+= varmod-remember
282 TESTS+= varmod-root
283 TESTS+= varmod-select-words
284 TESTS+= varmod-shell
285 TESTS+= varmod-subst
286 TESTS+= varmod-subst-regex
287 TESTS+= varmod-sysv
288 TESTS+= varmod-tail
289 TESTS+= varmod-to-abs
290 TESTS+= varmod-to-lower
291 TESTS+= varmod-to-many-words
292 TESTS+= varmod-to-one-word
293 TESTS+= varmod-to-separator
294 TESTS+= varmod-to-upper
295 TESTS+= varmod-undefined
296 TESTS+= varmod-unique
297 TESTS+= varname
298 TESTS+= varname-dollar
299 TESTS+= varname-dot-alltargets
300 TESTS+= varname-dot-curdir
301 TESTS+= varname-dot-includes
302 TESTS+= varname-dot-includedfromdir
303 TESTS+= varname-dot-includedfromfile
304 TESTS+= varname-dot-libs
305 TESTS+= varname-dot-make-dependfile
306 TESTS+= varname-dot-make-expand_variables
307 TESTS+= varname-dot-make-exported
308 TESTS+= varname-dot-make-jobs
309 TESTS+= varname-dot-make-jobs-prefix
310 TESTS+= varname-dot-make-level
311 TESTS+= varname-dot-make-makefile_preference
312 TESTS+= varname-dot-make-makefiles
313 TESTS+= varname-dot-make-meta-bailiwick
314 TESTS+= varname-dot-make-meta-created
315 TESTS+= varname-dot-make-meta-files
316 TESTS+= varname-dot-make-meta-ignore_filter
317 TESTS+= varname-dot-make-meta-ignore_paths
318 TESTS+= varname-dot-make-meta-ignore_patterns
319 TESTS+= varname-dot-make-meta-prefix
320 TESTS+= varname-dot-make-mode
321 TESTS+= varname-dot-make-path_filemon
322 TESTS+= varname-dot-make-pid
323 TESTS+= varname-dot-make-ppid
324 TESTS+= varname-dot-make-save_dollars
325 TESTS+= varname-dot-makeoverrides
326 TESTS+= varname-dot-newline
327 TESTS+= varname-dot-objdir
328 TESTS+= varname-dot-parsedir
329 TESTS+= varname-dot-parsefile
330 TESTS+= varname-dot-path
331 TESTS+= varname-dot-shell
332 TESTS+= varname-dot-targets
333 TESTS+= varname-empty
334 TESTS+= varname-make
335 TESTS+= varname-make_print_var_on_error
336 TESTS+= varname-makefile
337 TESTS+= varname-makeflags
338 TESTS+= varname-pwd
339 TESTS+= varname-vpath
340 TESTS+= varparse-dynamic
341 TESTS+= varquote
342 TESTS+= varshell
343
344 # Additional environment variables for some of the tests.
345 # The base environment is -i PATH="$PATH".
346 ENV.envfirst= FROM_ENV=value-from-env
347 ENV.varmisc= FROM_ENV=env
348 ENV.varmisc+= FROM_ENV_BEFORE=env
349 ENV.varmisc+= FROM_ENV_AFTER=env
350
351 # Override make flags for some of the tests; default is -k.
352 # If possible, write ".MAKEFLAGS: -dv" in the test .mk file instead of
353 # settings FLAGS.test=-dv here, since that is closer to the test code.
354 FLAGS.counter= -dv
355 FLAGS.directive-ifmake= first second
356 FLAGS.doterror= # none
357 FLAGS.envfirst= -e
358 FLAGS.export= # none
359 FLAGS.lint= -dL -k
360 FLAGS.opt-ignore= -i
361 FLAGS.opt-keep-going= -k
362 FLAGS.opt-no-action= -n
363 FLAGS.opt-query= -q
364 FLAGS.opt-var-expanded= -v VAR -v VALUE
365 FLAGS.opt-var-literal= -V VAR -V VALUE
366 FLAGS.opt-warnings-as-errors= -W
367 FLAGS.order= -j1
368 FLAGS.recursive= -dL
369 FLAGS.sh-leading-plus= -n
370 FLAGS.vardebug= -k -dv FROM_CMDLINE=
371 FLAGS.varmod-match-escape= -dv
372 FLAGS.varname-dot-shell= -dpv
373 FLAGS.varname-empty= -dv '$${:U}=cmdline-u' '=cmline-plain'
374
375 # Some tests need extra postprocessing.
376 SED_CMDS.opt-debug-graph1= \
377 -e 's,${.CURDIR},CURDIR,'
378 SED_CMDS.opt-debug-graph1+= \
379 -e '/Global Variables:/,/Suffixes:/d'
380 SED_CMDS.sh-dots= -e 's,^.*\.\.\.:.*,<normalized: ...: not found>,'
381 SED_CMDS.varmod-subst-regex+= \
382 -e 's,\(Regex compilation error:\).*,\1 (details omitted),'
383 SED_CMDS.varmod-edge+= -e 's, line [0-9]*:, line omitted:,'
384 SED_CMDS.varshell+= -e 's,^${.SHELL:T}: ,,'
385 SED_CMDS.varshell+= -e '/command/s,No such.*,not found,'
386 SED_CMDS.varname-dot-shell= -e 's, = /.*, = (details omitted),'
387 SED_CMDS.varname-dot-shell+= -e 's,"/[^"]*","(details omitted)",'
388 SED_CMDS.varname-dot-shell+= -e 's,\[/[^]]*\],[(details omitted)],'
389
390 # Some tests need an additional round of postprocessing.
391 POSTPROC.counter= ${TOOL_SED} -n -e '/:RELEVANT = yes/,/:RELEVANT = no/p'
392 POSTPROC.deptgt-suffixes= \
393 ${TOOL_SED} -n -e '/^\#\*\*\* Suffixes/,/^\#\*/p'
394 POSTPROC.vardebug= ${TOOL_SED} -n -e '/:RELEVANT = yes/,/:RELEVANT = no/p'
395 POSTPROC.varmod-match-escape= ${TOOL_SED} -n -e '/^Pattern/p'
396 POSTPROC.varname= ${TOOL_SED} -n -e '/^MAGIC/p' -e '/^ORDER_/p'
397 POSTPROC.varname-dot-shell= \
398 awk '/\.SHELL/ || /^ParseReadLine/'
399 POSTPROC.varname-empty= ${TOOL_SED} -n -e '/^Var_Set/p' -e '/^out:/p'
400
401 # Some tests reuse other tests, which makes them unnecessarily fragile.
402 export-all.rawout: export.mk
403 unexport.rawout: export.mk
404 unexport-env.rawout: export.mk
405
406 # End of the configuration section.
407
408 .MAIN: all
409
410 UNIT_TESTS:= ${.PARSEDIR}
411 .PATH: ${UNIT_TESTS}
412
413 OUTFILES= ${TESTS:=.out}
414
415 all: ${OUTFILES}
416
417 CLEANFILES= *.rawout *.out *.status *.tmp *.core *.tmp
418 CLEANFILES+= obj*.[och] lib*.a # posix1.mk
419 CLEANFILES+= issue* .[ab]* # suffixes.mk
420 CLEANDIRS= dir dummy # posix1.mk
421
422 clean:
423 rm -f ${CLEANFILES}
424 rm -rf ${CLEANDIRS}
425
426 TEST_MAKE?= ${.MAKE}
427 TOOL_SED?= sed
428
429 # ensure consistent results from sort(1)
430 LC_ALL= C
431 LANG= C
432 .export LANG LC_ALL
433
434 # Each test is run in a sub-make, to keep the tests for interfering with
435 # each other, and because they use different environment variables and
436 # command line options.
437 .SUFFIXES: .mk .rawout .out
438 .mk.rawout:
439 @${_MKMSG_TEST:Uecho '# test '} ${.PREFIX}
440 @set -eu; \
441 cd ${.OBJDIR}; \
442 env -i PATH="$$PATH" ${ENV.${.TARGET:R}} \
443 ${TEST_MAKE} \
444 -r -C ${.CURDIR} -f ${.IMPSRC} \
445 ${FLAGS.${.TARGET:R}:U-k} \
446 > ${.TARGET}.tmp 2>&1 \
447 && status=$$? || status=$$?; \
448 echo $$status > ${.TARGET:R}.status
449 @mv ${.TARGET}.tmp ${.TARGET}
450
451 # Postprocess the test output so that the results can be compared.
452 #
453 # always pretend .MAKE was called 'make'
454 _SED_CMDS+= -e 's,^${TEST_MAKE:T:S,.,\\.,g}[][0-9]*:,make:,'
455 _SED_CMDS+= -e 's,${TEST_MAKE:S,.,\\.,g},make,'
456 # replace anything after 'stopped in' with unit-tests
457 _SED_CMDS+= -e '/stopped/s, /.*, unit-tests,'
458 # strip ${.CURDIR}/ from the output
459 _SED_CMDS+= -e 's,${.CURDIR:S,.,\\.,g}/,,g'
460 _SED_CMDS+= -e 's,${UNIT_TESTS:S,.,\\.,g}/,,g'
461
462 .rawout.out:
463 @${TOOL_SED} ${_SED_CMDS} ${SED_CMDS.${.TARGET:R}} \
464 < ${.IMPSRC} > ${.TARGET}.tmp1
465 @${POSTPROC.${.TARGET:R}:Ucat} < ${.TARGET}.tmp1 > ${.TARGET}.tmp2
466 @rm ${.TARGET}.tmp1
467 @echo "exit status `cat ${.TARGET:R}.status`" >> ${.TARGET}.tmp2
468 @mv ${.TARGET}.tmp2 ${.TARGET}
469
470 # Compare all output files
471 test: ${OUTFILES} .PHONY
472 @failed= ; \
473 for test in ${TESTS}; do \
474 diff -u ${UNIT_TESTS}/$${test}.exp $${test}.out \
475 || failed="$${failed}$${failed:+ }$${test}" ; \
476 done ; \
477 if [ -n "$${failed}" ]; then \
478 echo "Failed tests: $${failed}" ; false ; \
479 else \
480 echo "All tests passed" ; \
481 fi
482
483 accept:
484 @for test in ${TESTS}; do \
485 cmp -s ${UNIT_TESTS}/$${test}.exp $${test}.out \
486 || { echo "Replacing $${test}.exp" ; \
487 cp $${test}.out ${UNIT_TESTS}/$${test}.exp ; } \
488 done
489
490 # Note: only works for adding tests.
491 # To remove a test, the $$mi file must be edited manually.
492 sync-mi:
493 @set -eu; \
494 cd "${MAKEFILE:tA:H}/../../.."; \
495 mi="distrib/sets/lists/tests/mi"; \
496 cvs update "$$mi"; \
497 testsdir="usr.bin/make/unit-tests"; \
498 fmt="./usr/tests/$$testsdir/%s\ttests-usr.bin-tests\tcompattestfile,atf\\n"; \
499 (cd "$$testsdir" && ls *.exp *.mk) | xargs printf "$$fmt" >> "$$mi"; \
500 distrib/sets/fmt-list "$$mi"; \
501 cvs diff "$$mi" || true
502
503 .if exists(${TEST_MAKE})
504 ${TESTS:=.rawout}: ${TEST_MAKE} ${.PARSEDIR}/Makefile
505 .endif
506
507 .-include <bsd.obj.mk>
508