1 #!/usr/bin/env python 2 3 import argparse 4 import sys 5 import os 6 7 from subprocess import call 8 9 SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__)) 10 PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects") 11 DEFAULT_LLVM_DIR = os.path.realpath(os.path.join(SCRIPTS_DIR, 12 os.path.pardir, 13 os.path.pardir, 14 os.path.pardir)) 15 16 17 def add(parser, args): 18 import SATestAdd 19 from ProjectMap import ProjectInfo 20 21 if args.source == "git" and (args.origin == "" or args.commit == ""): 22 parser.error( 23 "Please provide both --origin and --commit if source is 'git'") 24 25 if args.source != "git" and (args.origin != "" or args.commit != ""): 26 parser.error("Options --origin and --commit don't make sense when " 27 "source is not 'git'") 28 29 project = ProjectInfo(args.name[0], args.mode, args.source, args.origin, 30 args.commit) 31 32 SATestAdd.add_new_project(project) 33 34 35 def build(parser, args): 36 import SATestBuild 37 38 SATestBuild.VERBOSE = args.verbose 39 40 projects = get_projects(parser, args) 41 tester = SATestBuild.RegressionTester(args.jobs, 42 projects, 43 args.override_compiler, 44 args.extra_analyzer_config, 45 args.regenerate, 46 args.strictness) 47 tests_passed = tester.test_all() 48 49 if not tests_passed: 50 sys.stderr.write("ERROR: Tests failed.\n") 51 sys.exit(42) 52 53 54 def compare(parser, args): 55 import CmpRuns 56 57 choices = [CmpRuns.HistogramType.RELATIVE.value, 58 CmpRuns.HistogramType.LOG_RELATIVE.value, 59 CmpRuns.HistogramType.ABSOLUTE.value] 60 61 if args.histogram is not None and args.histogram not in choices: 62 parser.error("Incorrect histogram type, available choices are {}" 63 .format(choices)) 64 65 dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old) 66 dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new) 67 68 CmpRuns.dump_scan_build_results_diff(dir_old, dir_new, 69 show_stats=args.show_stats, 70 stats_only=args.stats_only, 71 histogram=args.histogram, 72 verbose_log=args.verbose_log) 73 74 75 def update(parser, args): 76 import SATestUpdateDiffs 77 from ProjectMap import ProjectMap 78 79 project_map = ProjectMap() 80 for project in project_map.projects: 81 SATestUpdateDiffs.update_reference_results(project, args.git) 82 83 84 def benchmark(parser, args): 85 from SATestBenchmark import Benchmark 86 87 projects = get_projects(parser, args) 88 benchmark = Benchmark(projects, args.iterations, args.output) 89 benchmark.run() 90 91 92 def benchmark_compare(parser, args): 93 import SATestBenchmark 94 SATestBenchmark.compare(args.old, args.new, args.output) 95 96 97 def get_projects(parser, args): 98 from ProjectMap import ProjectMap, Size 99 100 project_map = ProjectMap() 101 projects = project_map.projects 102 103 def filter_projects(projects, predicate, force=False): 104 return [project.with_fields(enabled=(force or project.enabled) and 105 predicate(project)) 106 for project in projects] 107 108 if args.projects: 109 projects_arg = args.projects.split(",") 110 available_projects = [project.name 111 for project in projects] 112 113 # validate that given projects are present in the project map file 114 for manual_project in projects_arg: 115 if manual_project not in available_projects: 116 parser.error("Project '{project}' is not found in " 117 "the project map file. Available projects are " 118 "{all}.".format(project=manual_project, 119 all=available_projects)) 120 121 projects = filter_projects(projects, lambda project: 122 project.name in projects_arg, 123 force=True) 124 125 try: 126 max_size = Size.from_str(args.max_size) 127 except ValueError as e: 128 parser.error("{}".format(e)) 129 130 projects = filter_projects(projects, lambda project: 131 project.size <= max_size) 132 133 return projects 134 135 136 def docker(parser, args): 137 if len(args.rest) > 0: 138 if args.rest[0] != "--": 139 parser.error("REST arguments should start with '--'") 140 args.rest = args.rest[1:] 141 142 if args.build_image: 143 docker_build_image() 144 elif args.shell: 145 docker_shell(args) 146 else: 147 sys.exit(docker_run(args, ' '.join(args.rest))) 148 149 150 def docker_build_image(): 151 sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), 152 shell=True)) 153 154 155 def docker_shell(args): 156 try: 157 # First we need to start the docker container in a waiting mode, 158 # so it doesn't do anything, but most importantly keeps working 159 # while the shell session is in progress. 160 docker_run(args, "--wait", "--detach") 161 # Since the docker container is running, we can actually connect to it 162 call("docker exec -it satest bash", shell=True) 163 164 except KeyboardInterrupt: 165 pass 166 167 finally: 168 docker_cleanup() 169 170 171 def docker_run(args, command, docker_args=""): 172 try: 173 return call("docker run --rm --name satest " 174 "-v {llvm}:/llvm-project " 175 "-v {build}:/build " 176 "-v {clang}:/analyzer " 177 "-v {scripts}:/scripts " 178 "-v {projects}:/projects " 179 "{docker_args} " 180 "satest-image:latest {command}" 181 .format(llvm=args.llvm_project_dir, 182 build=args.build_dir, 183 clang=args.clang_dir, 184 scripts=SCRIPTS_DIR, 185 projects=PROJECTS_DIR, 186 docker_args=docker_args, 187 command=command), 188 shell=True) 189 190 except KeyboardInterrupt: 191 docker_cleanup() 192 193 194 def docker_cleanup(): 195 print("Please wait for docker to clean up") 196 call("docker stop satest", shell=True) 197 198 199 def main(): 200 parser = argparse.ArgumentParser() 201 subparsers = parser.add_subparsers() 202 203 # add subcommand 204 add_parser = subparsers.add_parser( 205 "add", 206 help="Add a new project for the analyzer testing.") 207 # TODO: Add an option not to build. 208 # TODO: Set the path to the Repository directory. 209 add_parser.add_argument("name", nargs=1, help="Name of the new project") 210 add_parser.add_argument("--mode", action="store", default=1, type=int, 211 choices=[0, 1, 2], 212 help="Build mode: 0 for single file project, " 213 "1 for scan_build, " 214 "2 for single file c++11 project") 215 add_parser.add_argument("--source", action="store", default="script", 216 choices=["script", "git", "zip"], 217 help="Source type of the new project: " 218 "'git' for getting from git " 219 "(please provide --origin and --commit), " 220 "'zip' for unpacking source from a zip file, " 221 "'script' for downloading source by running " 222 "a custom script") 223 add_parser.add_argument("--origin", action="store", default="", 224 help="Origin link for a git repository") 225 add_parser.add_argument("--commit", action="store", default="", 226 help="Git hash for a commit to checkout") 227 add_parser.set_defaults(func=add) 228 229 # build subcommand 230 build_parser = subparsers.add_parser( 231 "build", 232 help="Build projects from the project map and compare results with " 233 "the reference.") 234 build_parser.add_argument("--strictness", dest="strictness", 235 type=int, default=0, 236 help="0 to fail on runtime errors, 1 to fail " 237 "when the number of found bugs are different " 238 "from the reference, 2 to fail on any " 239 "difference from the reference. Default is 0.") 240 build_parser.add_argument("-r", dest="regenerate", action="store_true", 241 default=False, 242 help="Regenerate reference output.") 243 build_parser.add_argument("--override-compiler", action="store_true", 244 default=False, help="Call scan-build with " 245 "--override-compiler option.") 246 build_parser.add_argument("-j", "--jobs", dest="jobs", 247 type=int, default=0, 248 help="Number of projects to test concurrently") 249 build_parser.add_argument("--extra-analyzer-config", 250 dest="extra_analyzer_config", type=str, 251 default="", 252 help="Arguments passed to to -analyzer-config") 253 build_parser.add_argument("--projects", action="store", default="", 254 help="Comma-separated list of projects to test") 255 build_parser.add_argument("--max-size", action="store", default=None, 256 help="Maximum size for the projects to test") 257 build_parser.add_argument("-v", "--verbose", action="count", default=0) 258 build_parser.set_defaults(func=build) 259 260 # compare subcommand 261 cmp_parser = subparsers.add_parser( 262 "compare", 263 help="Comparing two static analyzer runs in terms of " 264 "reported warnings and execution time statistics.") 265 cmp_parser.add_argument("--root-old", dest="root_old", 266 help="Prefix to ignore on source files for " 267 "OLD directory", 268 action="store", type=str, default="") 269 cmp_parser.add_argument("--root-new", dest="root_new", 270 help="Prefix to ignore on source files for " 271 "NEW directory", 272 action="store", type=str, default="") 273 cmp_parser.add_argument("--verbose-log", dest="verbose_log", 274 help="Write additional information to LOG " 275 "[default=None]", 276 action="store", type=str, default=None, 277 metavar="LOG") 278 cmp_parser.add_argument("--stats-only", action="store_true", 279 dest="stats_only", default=False, 280 help="Only show statistics on reports") 281 cmp_parser.add_argument("--show-stats", action="store_true", 282 dest="show_stats", default=False, 283 help="Show change in statistics") 284 cmp_parser.add_argument("--histogram", action="store", default=None, 285 help="Show histogram of paths differences. " 286 "Requires matplotlib") 287 cmp_parser.add_argument("old", nargs=1, help="Directory with old results") 288 cmp_parser.add_argument("new", nargs=1, help="Directory with new results") 289 cmp_parser.set_defaults(func=compare) 290 291 # update subcommand 292 upd_parser = subparsers.add_parser( 293 "update", 294 help="Update static analyzer reference results based on the previous " 295 "run of SATest build. Assumes that SATest build was just run.") 296 upd_parser.add_argument("--git", action="store_true", 297 help="Stage updated results using git.") 298 upd_parser.set_defaults(func=update) 299 300 # docker subcommand 301 dock_parser = subparsers.add_parser( 302 "docker", 303 help="Run regression system in the docker.") 304 305 dock_parser.add_argument("--build-image", action="store_true", 306 help="Build docker image for running tests.") 307 dock_parser.add_argument("--shell", action="store_true", 308 help="Start a shell on docker.") 309 dock_parser.add_argument("--llvm-project-dir", action="store", 310 default=DEFAULT_LLVM_DIR, 311 help="Path to LLVM source code. Defaults " 312 "to the repo where this script is located. ") 313 dock_parser.add_argument("--build-dir", action="store", default="", 314 help="Path to a directory where docker should " 315 "build LLVM code.") 316 dock_parser.add_argument("--clang-dir", action="store", default="", 317 help="Path to find/install LLVM installation.") 318 dock_parser.add_argument("rest", nargs=argparse.REMAINDER, default=[], 319 help="Additionall args that will be forwarded " 320 "to the docker's entrypoint.") 321 dock_parser.set_defaults(func=docker) 322 323 # benchmark subcommand 324 bench_parser = subparsers.add_parser( 325 "benchmark", 326 help="Run benchmarks by building a set of projects multiple times.") 327 328 bench_parser.add_argument("-i", "--iterations", action="store", 329 type=int, default=20, 330 help="Number of iterations for building each " 331 "project.") 332 bench_parser.add_argument("-o", "--output", action="store", 333 default="benchmark.csv", 334 help="Output csv file for the benchmark results") 335 bench_parser.add_argument("--projects", action="store", default="", 336 help="Comma-separated list of projects to test") 337 bench_parser.add_argument("--max-size", action="store", default=None, 338 help="Maximum size for the projects to test") 339 bench_parser.set_defaults(func=benchmark) 340 341 bench_subparsers = bench_parser.add_subparsers() 342 bench_compare_parser = bench_subparsers.add_parser( 343 "compare", 344 help="Compare benchmark runs.") 345 bench_compare_parser.add_argument("--old", action="store", required=True, 346 help="Benchmark reference results to " 347 "compare agains.") 348 bench_compare_parser.add_argument("--new", action="store", required=True, 349 help="New benchmark results to check.") 350 bench_compare_parser.add_argument("-o", "--output", 351 action="store", required=True, 352 help="Output file for plots.") 353 bench_compare_parser.set_defaults(func=benchmark_compare) 354 355 args = parser.parse_args() 356 args.func(parser, args) 357 358 359 if __name__ == "__main__": 360 main() 361
Indexes created Tue Feb 24 08:35:24 UTC 2026