test-variants.mk revision 1.2
1 # $NetBSD: test-variants.mk,v 1.2 2022/03/03 20:03:19 rillig Exp $ 2 # 3 # Build several variants of make and run the tests on them. 4 # 5 # The output of this test suite must be inspected manually to see the 6 # interesting details. The main purpose is to list the available build 7 # options. 8 9 .MAIN: usage 10 usage: 11 @echo 'usage: ${MAKE} -f ${MAKEFILE} list' 12 @echo ' ${MAKE} -f ${MAKEFILE} all' 13 @echo ' ${MAKE} -f ${MAKEFILE} test...' 14 15 16 TESTS+= default 17 #ENV.default= VAR=value... 18 #CPPFLAGS.default= -DMACRO -I... 19 #CFLAGS.default= -O1 20 #SKIP.default= yes 21 #SKIP_TESTS.default= varmod-subst 22 23 # Try a different compiler, with slightly different warnings and error 24 # messages. Clang has a few stricter checks than GCC, concerning enums 25 # and non-literal format strings. 26 # 27 TESTS+= llvm 28 ENV.llvm= HAVE_LLVM="yes" 29 30 TESTS+= gcc8 31 ENV.gcc8= USE_GCC8="yes" 32 33 TESTS+= gcc9 34 ENV.gcc9= USE_GCC9="yes" 35 36 # Use emalloc for memory allocation. 37 TESTS+= emalloc 38 ENV.emalloc= TOOLDIR="" 39 40 # NetBSD 10 fails with: 41 # filemon_dev.c:93:19: error: 'FILEMON_SET_FD' undeclared 42 TESTS+= filemon-dev 43 ENV.filemon-dev= USE_FILEMON="dev" 44 SKIP.filemon-dev= yes 45 46 TESTS+= filemon-ktrace 47 ENV.filemon-ktrace= USE_FILEMON="ktrace" 48 49 TESTS+= filemon-none 50 ENV.filemon-none= USE_FILEMON="no" 51 # The following tests only fail due to the extra variable in the debug log. 52 SKIP_TESTS.filemon-none= \ 53 opt-debug-graph1 \ 54 opt-debug-graph2 \ 55 opt-debug-graph3 \ 56 suff-main-several \ 57 suff-transform-debug 58 59 TESTS+= no-meta 60 ENV.no-meta= USE_META="no" 61 SKIP_TESTS.no-meta= depsrc-meta meta-cmd-cmp 62 63 TESTS+= cleanup 64 CPPFLAGS.cleanup= -DCLEANUP 65 66 TESTS+= debug-refcnt 67 CPPFLAGS.debug-refcnt= -DDEBUG_REFCNT 68 69 TESTS+= debug-hash 70 CPPFLAGS.debug-hash= -DDEBUG_HASH_LOOKUP 71 SKIP_TESTS.debug-hash= opt-debug-hash # mixes regular and debug output 72 73 TESTS+= debug-meta 74 CPPFLAGS.debug-meta= -DDEBUG_META_MODE 75 SKIP_TESTS.debug-meta= depsrc-meta meta-cmd-cmp # generate extra debug output 76 77 # Produces lots of debugging output. 78 # 79 TESTS+= debug-src 80 CPPFLAGS.debug-src= -DDEBUG_SRC 81 SKIP.debug-src= yes 82 83 # In this variant, the tests for 'export VAR=value' fail, as expected. 84 # 85 TESTS+= no-gmake-export 86 CPPFLAGS.no-gmake-export= -UGMAKEEXPORT 87 SKIP.no-gmake-export= yes 88 89 # NetBSD 8.0 x86_64 90 # In file included from arch.c:135:0: 91 # /usr/include/sys/param.h:357:0: error: "MAXPATHLEN" redefined [-Werror] 92 TESTS+= maxpathlen 93 CPPFLAGS.maxpathlen= -DMAXPATHLEN=20 94 SKIP.maxpathlen= yes 95 96 # In this variant, the unit tests using the modifier ':C' fail, as expected. 97 # 98 TESTS+= no-regex 99 CPPFLAGS.no-regex= -DNO_REGEX 100 SKIP_TESTS.no-regex= archive cond-short deptgt-makeflags dollar export-all 101 SKIP_TESTS.no-regex+= moderrs modmatch modmisc var-eval-short 102 SKIP_TESTS.no-regex+= varmod-select-words varmod-subst varmod-subst-regex 103 SKIP_TESTS.no-regex+= varname-dot-make-pid varname-dot-make-ppid 104 105 # NetBSD 8.0 x86_64 says: 106 # In file included from /usr/include/sys/param.h:115:0, 107 # from arch.c:135: 108 # /usr/include/sys/syslimits.h:60:0: error: "PATH_MAX" redefined [-Werror] 109 TESTS+= path_max 110 CPPFLAGS.path_max= -DPATH_MAX=20 111 SKIP.path_max= yes 112 113 # This higher optimization level may trigger additional "may be used 114 # uninitialized" errors. Could be combined with other compilers as well. 115 # 116 TESTS+= opt-3 117 CFLAGS.opt-3= -O3 118 119 # When optimizing for small code size, GCC gets confused by the initialization 120 # status of local variables in some cases. 121 TESTS+= opt-size 122 CFLAGS.opt-size= -Os 123 124 TESTS+= opt-none 125 CFLAGS.opt-none= -O0 -ggdb 126 127 # Ensure that every inline function is declared as MAKE_ATTR_UNUSED. 128 TESTS+= no-inline 129 CPPFLAGS.no-inline= -Dinline= 130 131 # Is expected to fail with "<stdbool.h> is included in pre-C99 mode" since 132 # main.c includes <sys/sysctl.h>, which includes <stdbool.h> even in pre-C99 133 # mode. This would lead to inconsistent definitions of bool and thus 134 # differently sized struct CmdOpts and others. 135 TESTS+= c90-plain 136 ENV.c90-plain= USE_FILEMON=no # filemon uses designated initializers 137 CFLAGS.c90-plain= -std=c90 -ansi -pedantic -Wno-system-headers 138 SKIP.c90-plain= yes 139 140 # The make source code is _intended_ to be compatible with C90, it uses some 141 # C99 features though (snprintf). The workaround with USE_C99_BOOLEAN is 142 # necessary on NetBSD 9.99 since main.c includes <sys/sysctl.h>, which 143 # includes <stdbool.h> even in pre-C99 mode. 144 TESTS+= c90-stdbool 145 ENV.c90-stdbool= USE_FILEMON=no # filemon uses designated initializers 146 CFLAGS.c90-stdbool= -std=c90 -ansi -pedantic -Wno-system-headers 147 CPPFLAGS.c90-stdbool= -DUSE_C99_BOOLEAN 148 149 # Ensure that there are only side-effect-free conditions in the assert 150 # macro, or at least none that affect the outcome of the tests. 151 # 152 TESTS+= no-assert 153 CPPFLAGS.no-assert= -DNDEBUG 154 155 # Only in native mode, make dares to use a shortcut in Compat_RunCommand 156 # that circumvents the shell and instead calls execvp directly. 157 # Another effect is that the shell is run with -q, which prevents the 158 # -x and -v flags from echoing the commands from profile files. 159 TESTS+= non-native 160 CPPFLAGS.non-native= -UMAKE_NATIVE 161 CPPFLAGS.non-native+= -DHAVE_STRERROR -DHAVE_SETENV -DHAVE_VSNPRINTF 162 163 # Running the code coverage using gcov took a long time on NetBSD < 10, due to 164 # https://gnats.netbsd.org/55808. 165 # 166 # Combining USE_COVERAGE with HAVE_LLVM does not work since ld fails to link 167 # with the coverage library. 168 # 169 # Turning the optimization off is required because gcov does not work on the 170 # source code level but on the intermediate code after optimization: 171 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96622 172 # 173 TESTS+= coverage 174 ENV.coverage= USE_COVERAGE="yes" 175 CFLAGS.coverage= -O0 -ggdb 176 177 TESTS+= fort 178 ENV.fort= USE_FORT="yes" 179 180 # Ensure that the tests can be specified either as relative filenames or 181 # as absolute filenames. 182 TESTS+= abs 183 ENV.abs= USE_ABSOLUTE_TESTNAMES="yes" 184 185 # This test is the result of reading through the GCC "Warning Options" 186 # documentation, noting down everything that sounded interesting. 187 # 188 TESTS+= gcc10-warn 189 ENV.gcc10-warn= USE_GCC10=yes 190 CFLAGS.gcc10-warn= -Wmisleading-indentation 191 CFLAGS.gcc10-warn+= -Wmissing-attributes 192 CFLAGS.gcc10-warn+= -Wmissing-braces 193 CFLAGS.gcc10-warn+= -Wextra 194 CFLAGS.gcc10-warn+= -Wnonnull 195 CFLAGS.gcc10-warn+= -Wnonnull-compare 196 CFLAGS.gcc10-warn+= -Wnull-dereference 197 CFLAGS.gcc10-warn+= -Wimplicit 198 CFLAGS.gcc10-warn+= -Wimplicit-fallthrough=4 199 CFLAGS.gcc10-warn+= -Wignored-qualifiers 200 CFLAGS.gcc10-warn+= -Wunused 201 CFLAGS.gcc10-warn+= -Wunused-but-set-variable 202 CFLAGS.gcc10-warn+= -Wunused-parameter 203 CFLAGS.gcc10-warn+= -Wduplicated-branches 204 CFLAGS.gcc10-warn+= -Wduplicated-cond 205 CFLAGS.gcc10-warn+= -Wunused-macros 206 CFLAGS.gcc10-warn+= -Wcast-function-type 207 CFLAGS.gcc10-warn+= -Wconversion 208 CFLAGS.gcc10-warn+= -Wenum-compare 209 CFLAGS.gcc10-warn+= -Wmissing-field-initializers 210 CFLAGS.gcc10-warn+= -Wredundant-decls 211 CFLAGS.gcc10-warn+= -Wno-error=conversion 212 CFLAGS.gcc10-warn+= -Wno-error=sign-conversion 213 CFLAGS.gcc10-warn+= -Wno-error=unused-macros 214 CFLAGS.gcc10-warn+= -Wno-error=unused-parameter 215 CFLAGS.gcc10-warn+= -Wno-error=duplicated-branches 216 217 .for shell in /usr/pkg/bin/bash /usr/pkg/bin/dash 218 . if exists(${shell}) 219 TESTS+= ${shell:T} 220 CPPFLAGS.${shell:T}= -DDEFSHELL_CUSTOM="\"${shell}\"" 221 . endif 222 .endfor 223 224 225 .MAKEFLAGS: -k 226 .MAKE.DEPENDFILE= nonexistent 227 228 .PHONY: usage list all ${TESTS} 229 230 all: ${TESTS:${TESTS:@t@${SKIP.$t:Myes:%=N$t}@:ts:}} 231 232 SLOW_TESTS= dotwait sh-flags 233 234 .for test in ${TESTS} 235 _ENV.${test}= PATH="$$PATH" 236 _ENV.${test}+= USETOOLS="no" 237 _ENV.${test}+= MALLOC_OPTIONS="JA" # for jemalloc 1.0.0 238 _ENV.${test}+= MALLOC_CONF="junk:true" # for jemalloc 5.1.0 239 _ENV.${test}+= _MKMSG_COMPILE=":" 240 _ENV.${test}+= _MKMSG_CREATE=":" 241 _ENV.${test}+= _MKMSG_FORMAT=":" 242 _ENV.${test}+= _MKMSG_TEST=":" 243 _ENV.${test}+= ${ENV.${test}} 244 _ENV.${test}+= USER_CPPFLAGS=${CPPFLAGS.${.TARGET}:Q} 245 _ENV.${test}+= USER_CFLAGS=${CFLAGS.${.TARGET}:Q} 246 _ENV.${test}+= BROKEN_TESTS=${${SLOW_TESTS} ${SKIP_TESTS.${.TARGET}}:L:Q} 247 .endfor 248 249 ${TESTS}: run-test 250 run-test: .USE 251 @echo "===> Running ${.TARGET}" 252 @echo "env: "${ENV.${.TARGET}:U"(none)"} 253 @echo "cflags: "${CFLAGS.${.TARGET}:U(none):Q} 254 @echo "cppflags: "${CPPFLAGS.${.TARGET}:U(none):Q} 255 256 @env -i ${_ENV.${.TARGET}} sh -ce "ma""ke -s cleandir" 257 @env -i ${_ENV.${.TARGET}} sh -ce "ma""ke -ks -j6 dependall" 258 @size *.o make 259 @env -i ${_ENV.${.TARGET}} sh -ce "ma""ke -s test" 260 261 list: 262 @printf '%s%s\n' ${TESTS:O:@t@$t '${SKIP.$t:Myes:%= (skipped)}'@} 263
Indexes created Thu Oct 30 03:10:01 GMT 2025