1 1.1 christos #!/usr/bin/env python3 2 1.1 christos # ################################################################ 3 1.1 christos # Copyright (c) Meta Platforms, Inc. and affiliates. 4 1.1 christos # All rights reserved. 5 1.1 christos # 6 1.1 christos # This source code is licensed under both the BSD-style license (found in the 7 1.1 christos # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 1.1 christos # in the COPYING file in the root directory of this source tree). 9 1.1 christos # You may select, at your option, one of the above-listed licenses. 10 1.1 christos # ########################################################################## 11 1.1 christos 12 1.1 christos import argparse 13 1.1 christos import contextlib 14 1.1 christos import os 15 1.1 christos import re 16 1.1 christos import shutil 17 1.1 christos import sys 18 1.1 christos from typing import Optional 19 1.1 christos 20 1.1 christos 21 1.1 christos INCLUDED_SUBDIRS = ["common", "compress", "decompress"] 22 1.1 christos 23 1.1 christos SKIPPED_FILES = [ 24 1.1 christos "common/mem.h", 25 1.1 christos "common/zstd_deps.h", 26 1.1 christos "common/pool.c", 27 1.1 christos "common/pool.h", 28 1.1 christos "common/threading.c", 29 1.1 christos "common/threading.h", 30 1.1 christos "common/zstd_trace.h", 31 1.1 christos "compress/zstdmt_compress.h", 32 1.1 christos "compress/zstdmt_compress.c", 33 1.1 christos ] 34 1.1 christos 35 1.1 christos XXHASH_FILES = [ 36 1.1 christos "common/xxhash.c", 37 1.1 christos "common/xxhash.h", 38 1.1 christos ] 39 1.1 christos 40 1.1 christos 41 1.1 christos class FileLines(object): 42 1.1 christos def __init__(self, filename): 43 1.1 christos self.filename = filename 44 1.1 christos with open(self.filename, "r") as f: 45 1.1 christos self.lines = f.readlines() 46 1.1 christos 47 1.1 christos def write(self): 48 1.1 christos with open(self.filename, "w") as f: 49 1.1 christos f.write("".join(self.lines)) 50 1.1 christos 51 1.1 christos 52 1.1 christos class PartialPreprocessor(object): 53 1.1 christos """ 54 1.1 christos Looks for simple ifdefs and ifndefs and replaces them. 55 1.1 christos Handles && and ||. 56 1.1 christos Has fancy logic to handle translating elifs to ifs. 57 1.1 christos Only looks for macros in the first part of the expression with no 58 1.1 christos parens. 59 1.1 christos Does not handle multi-line macros (only looks in first line). 60 1.1 christos """ 61 1.1 christos def __init__(self, defs: [(str, Optional[str])], replaces: [(str, str)], undefs: [str]): 62 1.1 christos MACRO_GROUP = r"(?P<macro>[a-zA-Z_][a-zA-Z_0-9]*)" 63 1.1 christos ELIF_GROUP = r"(?P<elif>el)?" 64 1.1 christos OP_GROUP = r"(?P<op>&&|\|\|)?" 65 1.1 christos 66 1.1 christos self._defs = {macro:value for macro, value in defs} 67 1.1 christos self._replaces = {macro:value for macro, value in replaces} 68 1.1 christos self._defs.update(self._replaces) 69 1.1 christos self._undefs = set(undefs) 70 1.1 christos 71 1.1 christos self._define = re.compile(r"\s*#\s*define") 72 1.1 christos self._if = re.compile(r"\s*#\s*if") 73 1.1 christos self._elif = re.compile(r"\s*#\s*(?P<elif>el)if") 74 1.1 christos self._else = re.compile(r"\s*#\s*(?P<else>else)") 75 1.1 christos self._endif = re.compile(r"\s*#\s*endif") 76 1.1 christos 77 1.1 christos self._ifdef = re.compile(fr"\s*#\s*if(?P<not>n)?def {MACRO_GROUP}\s*") 78 1.1 christos self._if_defined = re.compile( 79 1.1 christos fr"\s*#\s*{ELIF_GROUP}if\s+(?P<not>!)?\s*defined\s*\(\s*{MACRO_GROUP}\s*\)\s*{OP_GROUP}" 80 1.1 christos ) 81 1.1 christos self._if_defined_value = re.compile( 82 1.1 christos fr"\s*#\s*{ELIF_GROUP}if\s+defined\s*\(\s*{MACRO_GROUP}\s*\)\s*" 83 1.1 christos fr"(?P<op>&&)\s*" 84 1.1 christos fr"(?P<openp>\()?\s*" 85 1.1 christos fr"(?P<macro2>[a-zA-Z_][a-zA-Z_0-9]*)\s*" 86 1.1 christos fr"(?P<cmp>[=><!]+)\s*" 87 1.1 christos fr"(?P<value>[0-9]*)\s*" 88 1.1 christos fr"(?P<closep>\))?\s*" 89 1.1 christos ) 90 1.1 christos self._if_true = re.compile( 91 1.1 christos fr"\s*#\s*{ELIF_GROUP}if\s+{MACRO_GROUP}\s*{OP_GROUP}" 92 1.1 christos ) 93 1.1 christos 94 1.1 christos self._c_comment = re.compile(r"/\*.*?\*/") 95 1.1 christos self._cpp_comment = re.compile(r"//") 96 1.1 christos 97 1.1 christos def _log(self, *args, **kwargs): 98 1.1 christos print(*args, **kwargs) 99 1.1 christos 100 1.1 christos def _strip_comments(self, line): 101 1.1 christos # First strip c-style comments (may include //) 102 1.1 christos while True: 103 1.1 christos m = self._c_comment.search(line) 104 1.1 christos if m is None: 105 1.1 christos break 106 1.1 christos line = line[:m.start()] + line[m.end():] 107 1.1 christos 108 1.1 christos # Then strip cpp-style comments 109 1.1 christos m = self._cpp_comment.search(line) 110 1.1 christos if m is not None: 111 1.1 christos line = line[:m.start()] 112 1.1 christos 113 1.1 christos return line 114 1.1 christos 115 1.1 christos def _fixup_indentation(self, macro, replace: [str]): 116 1.1 christos if len(replace) == 0: 117 1.1 christos return replace 118 1.1 christos if len(replace) == 1 and self._define.match(replace[0]) is None: 119 1.1 christos # If there is only one line, only replace defines 120 1.1 christos return replace 121 1.1 christos 122 1.1 christos 123 1.1 christos all_pound = True 124 1.1 christos for line in replace: 125 1.1 christos if not line.startswith('#'): 126 1.1 christos all_pound = False 127 1.1 christos if all_pound: 128 1.1 christos replace = [line[1:] for line in replace] 129 1.1 christos 130 1.1 christos min_spaces = len(replace[0]) 131 1.1 christos for line in replace: 132 1.1 christos spaces = 0 133 1.1 christos for i, c in enumerate(line): 134 1.1 christos if c != ' ': 135 1.1 christos # Non-preprocessor line ==> skip the fixup 136 1.1 christos if not all_pound and c != '#': 137 1.1 christos return replace 138 1.1 christos spaces = i 139 1.1 christos break 140 1.1 christos min_spaces = min(min_spaces, spaces) 141 1.1 christos 142 1.1 christos replace = [line[min_spaces:] for line in replace] 143 1.1 christos 144 1.1 christos if all_pound: 145 1.1 christos replace = ["#" + line for line in replace] 146 1.1 christos 147 1.1 christos return replace 148 1.1 christos 149 1.1 christos def _handle_if_block(self, macro, idx, is_true, prepend): 150 1.1 christos """ 151 1.1 christos Remove the #if or #elif block starting on this line. 152 1.1 christos """ 153 1.1 christos REMOVE_ONE = 0 154 1.1 christos KEEP_ONE = 1 155 1.1 christos REMOVE_REST = 2 156 1.1 christos 157 1.1 christos if is_true: 158 1.1 christos state = KEEP_ONE 159 1.1 christos else: 160 1.1 christos state = REMOVE_ONE 161 1.1 christos 162 1.1 christos line = self._inlines[idx] 163 1.1 christos is_if = self._if.match(line) is not None 164 1.1 christos assert is_if or self._elif.match(line) is not None 165 1.1 christos depth = 0 166 1.1 christos 167 1.1 christos start_idx = idx 168 1.1 christos 169 1.1 christos idx += 1 170 1.1 christos replace = prepend 171 1.1 christos finished = False 172 1.1 christos while idx < len(self._inlines): 173 1.1 christos line = self._inlines[idx] 174 1.1 christos # Nested if statement 175 1.1 christos if self._if.match(line): 176 1.1 christos depth += 1 177 1.1 christos idx += 1 178 1.1 christos continue 179 1.1 christos # We're inside a nested statement 180 1.1 christos if depth > 0: 181 1.1 christos if self._endif.match(line): 182 1.1 christos depth -= 1 183 1.1 christos idx += 1 184 1.1 christos continue 185 1.1 christos 186 1.1 christos # We're at the original depth 187 1.1 christos 188 1.1 christos # Looking only for an endif. 189 1.1 christos # We've found a true statement, but haven't 190 1.1 christos # completely elided the if block, so we just 191 1.1 christos # remove the remainder. 192 1.1 christos if state == REMOVE_REST: 193 1.1 christos if self._endif.match(line): 194 1.1 christos if is_if: 195 1.1 christos # Remove the endif because we took the first if 196 1.1 christos idx += 1 197 1.1 christos finished = True 198 1.1 christos break 199 1.1 christos idx += 1 200 1.1 christos continue 201 1.1 christos 202 1.1 christos if state == KEEP_ONE: 203 1.1 christos m = self._elif.match(line) 204 1.1 christos if self._endif.match(line): 205 1.1 christos replace += self._inlines[start_idx + 1:idx] 206 1.1 christos idx += 1 207 1.1 christos finished = True 208 1.1 christos break 209 1.1 christos if self._elif.match(line) or self._else.match(line): 210 1.1 christos replace += self._inlines[start_idx + 1:idx] 211 1.1 christos state = REMOVE_REST 212 1.1 christos idx += 1 213 1.1 christos continue 214 1.1 christos 215 1.1 christos if state == REMOVE_ONE: 216 1.1 christos m = self._elif.match(line) 217 1.1 christos if m is not None: 218 1.1 christos if is_if: 219 1.1 christos idx += 1 220 1.1 christos b = m.start('elif') 221 1.1 christos e = m.end('elif') 222 1.1 christos assert e - b == 2 223 1.1 christos replace.append(line[:b] + line[e:]) 224 1.1 christos finished = True 225 1.1 christos break 226 1.1 christos m = self._else.match(line) 227 1.1 christos if m is not None: 228 1.1 christos if is_if: 229 1.1 christos idx += 1 230 1.1 christos while self._endif.match(self._inlines[idx]) is None: 231 1.1 christos replace.append(self._inlines[idx]) 232 1.1 christos idx += 1 233 1.1 christos idx += 1 234 1.1 christos finished = True 235 1.1 christos break 236 1.1 christos if self._endif.match(line): 237 1.1 christos if is_if: 238 1.1 christos # Remove the endif because no other elifs 239 1.1 christos idx += 1 240 1.1 christos finished = True 241 1.1 christos break 242 1.1 christos idx += 1 243 1.1 christos continue 244 1.1 christos if not finished: 245 1.1 christos raise RuntimeError("Unterminated if block!") 246 1.1 christos 247 1.1 christos replace = self._fixup_indentation(macro, replace) 248 1.1 christos 249 1.1 christos self._log(f"\tHardwiring {macro}") 250 1.1 christos if start_idx > 0: 251 1.1 christos self._log(f"\t\t {self._inlines[start_idx - 1][:-1]}") 252 1.1 christos for x in range(start_idx, idx): 253 1.1 christos self._log(f"\t\t- {self._inlines[x][:-1]}") 254 1.1 christos for line in replace: 255 1.1 christos self._log(f"\t\t+ {line[:-1]}") 256 1.1 christos if idx < len(self._inlines): 257 1.1 christos self._log(f"\t\t {self._inlines[idx][:-1]}") 258 1.1 christos 259 1.1 christos return idx, replace 260 1.1 christos 261 1.1 christos def _preprocess_once(self): 262 1.1 christos outlines = [] 263 1.1 christos idx = 0 264 1.1 christos changed = False 265 1.1 christos while idx < len(self._inlines): 266 1.1 christos line = self._inlines[idx] 267 1.1 christos sline = self._strip_comments(line) 268 1.1 christos m = self._ifdef.fullmatch(sline) 269 1.1 christos if_true = False 270 1.1 christos if m is None: 271 1.1 christos m = self._if_defined_value.fullmatch(sline) 272 1.1 christos if m is None: 273 1.1 christos m = self._if_defined.match(sline) 274 1.1 christos if m is None: 275 1.1 christos m = self._if_true.match(sline) 276 1.1 christos if_true = (m is not None) 277 1.1 christos if m is None: 278 1.1 christos outlines.append(line) 279 1.1 christos idx += 1 280 1.1 christos continue 281 1.1 christos 282 1.1 christos groups = m.groupdict() 283 1.1 christos macro = groups['macro'] 284 1.1 christos op = groups.get('op') 285 1.1 christos 286 1.1 christos if not (macro in self._defs or macro in self._undefs): 287 1.1 christos outlines.append(line) 288 1.1 christos idx += 1 289 1.1 christos continue 290 1.1 christos 291 1.1 christos defined = macro in self._defs 292 1.1 christos 293 1.1 christos # Needed variables set: 294 1.1 christos # resolved: Is the statement fully resolved? 295 1.1 christos # is_true: If resolved, is the statement true? 296 1.1 christos ifdef = False 297 1.1 christos if if_true: 298 1.1 christos if not defined: 299 1.1 christos outlines.append(line) 300 1.1 christos idx += 1 301 1.1 christos continue 302 1.1 christos 303 1.1 christos defined_value = self._defs[macro] 304 1.1 christos is_int = True 305 1.1 christos try: 306 1.1 christos defined_value = int(defined_value) 307 1.1 christos except TypeError: 308 1.1 christos is_int = False 309 1.1 christos except ValueError: 310 1.1 christos is_int = False 311 1.1 christos 312 1.1 christos resolved = is_int 313 1.1 christos is_true = (defined_value != 0) 314 1.1 christos 315 1.1 christos if resolved and op is not None: 316 1.1 christos if op == '&&': 317 1.1 christos resolved = not is_true 318 1.1 christos else: 319 1.1 christos assert op == '||' 320 1.1 christos resolved = is_true 321 1.1 christos 322 1.1 christos else: 323 1.1 christos ifdef = groups.get('not') is None 324 1.1 christos elseif = groups.get('elif') is not None 325 1.1 christos 326 1.1 christos macro2 = groups.get('macro2') 327 1.1 christos cmp = groups.get('cmp') 328 1.1 christos value = groups.get('value') 329 1.1 christos openp = groups.get('openp') 330 1.1 christos closep = groups.get('closep') 331 1.1 christos 332 1.1 christos is_true = (ifdef == defined) 333 1.1 christos resolved = True 334 1.1 christos if op is not None: 335 1.1 christos if op == '&&': 336 1.1 christos resolved = not is_true 337 1.1 christos else: 338 1.1 christos assert op == '||' 339 1.1 christos resolved = is_true 340 1.1 christos 341 1.1 christos if macro2 is not None and not resolved: 342 1.1 christos assert ifdef and defined and op == '&&' and cmp is not None 343 1.1 christos # If the statement is true, but we have a single value check, then 344 1.1 christos # check the value. 345 1.1 christos defined_value = self._defs[macro] 346 1.1 christos are_ints = True 347 1.1 christos try: 348 1.1 christos defined_value = int(defined_value) 349 1.1 christos value = int(value) 350 1.1 christos except TypeError: 351 1.1 christos are_ints = False 352 1.1 christos except ValueError: 353 1.1 christos are_ints = False 354 1.1 christos if ( 355 1.1 christos macro == macro2 and 356 1.1 christos ((openp is None) == (closep is None)) and 357 1.1 christos are_ints 358 1.1 christos ): 359 1.1 christos resolved = True 360 1.1 christos if cmp == '<': 361 1.1 christos is_true = defined_value < value 362 1.1 christos elif cmp == '<=': 363 1.1 christos is_true = defined_value <= value 364 1.1 christos elif cmp == '==': 365 1.1 christos is_true = defined_value == value 366 1.1 christos elif cmp == '!=': 367 1.1 christos is_true = defined_value != value 368 1.1 christos elif cmp == '>=': 369 1.1 christos is_true = defined_value >= value 370 1.1 christos elif cmp == '>': 371 1.1 christos is_true = defined_value > value 372 1.1 christos else: 373 1.1 christos resolved = False 374 1.1 christos 375 1.1 christos if op is not None and not resolved: 376 1.1 christos # Remove the first op in the line + spaces 377 1.1 christos if op == '&&': 378 1.1 christos opre = op 379 1.1 christos else: 380 1.1 christos assert op == '||' 381 1.1 christos opre = r'\|\|' 382 1.1 christos needle = re.compile(fr"(?P<if>\s*#\s*(el)?if\s+).*?(?P<op>{opre}\s*)") 383 1.1 christos match = needle.match(line) 384 1.1 christos assert match is not None 385 1.1 christos newline = line[:match.end('if')] + line[match.end('op'):] 386 1.1 christos 387 1.1 christos self._log(f"\tHardwiring partially resolved {macro}") 388 1.1 christos self._log(f"\t\t- {line[:-1]}") 389 1.1 christos self._log(f"\t\t+ {newline[:-1]}") 390 1.1 christos 391 1.1 christos outlines.append(newline) 392 1.1 christos idx += 1 393 1.1 christos continue 394 1.1 christos 395 1.1 christos # Skip any statements we cannot fully compute 396 1.1 christos if not resolved: 397 1.1 christos outlines.append(line) 398 1.1 christos idx += 1 399 1.1 christos continue 400 1.1 christos 401 1.1 christos prepend = [] 402 1.1 christos if macro in self._replaces: 403 1.1 christos assert not ifdef 404 1.1 christos assert op is None 405 1.1 christos value = self._replaces.pop(macro) 406 1.1 christos prepend = [f"#define {macro} {value}\n"] 407 1.1 christos 408 1.1 christos idx, replace = self._handle_if_block(macro, idx, is_true, prepend) 409 1.1 christos outlines += replace 410 1.1 christos changed = True 411 1.1 christos 412 1.1 christos return changed, outlines 413 1.1 christos 414 1.1 christos def preprocess(self, filename): 415 1.1 christos with open(filename, 'r') as f: 416 1.1 christos self._inlines = f.readlines() 417 1.1 christos changed = True 418 1.1 christos iters = 0 419 1.1 christos while changed: 420 1.1 christos iters += 1 421 1.1 christos changed, outlines = self._preprocess_once() 422 1.1 christos self._inlines = outlines 423 1.1 christos 424 1.1 christos with open(filename, 'w') as f: 425 1.1 christos f.write(''.join(self._inlines)) 426 1.1 christos 427 1.1 christos 428 1.1 christos class Freestanding(object): 429 1.1 christos def __init__( 430 1.1 christos self, zstd_deps: str, mem: str, source_lib: str, output_lib: str, 431 1.1 christos external_xxhash: bool, xxh64_state: Optional[str], 432 1.1 christos xxh64_prefix: Optional[str], rewritten_includes: [(str, str)], 433 1.1 christos defs: [(str, Optional[str])], replaces: [(str, str)], 434 1.1 christos undefs: [str], excludes: [str], seds: [str], spdx: bool, 435 1.1 christos ): 436 1.1 christos self._zstd_deps = zstd_deps 437 1.1 christos self._mem = mem 438 1.1 christos self._src_lib = source_lib 439 1.1 christos self._dst_lib = output_lib 440 1.1 christos self._external_xxhash = external_xxhash 441 1.1 christos self._xxh64_state = xxh64_state 442 1.1 christos self._xxh64_prefix = xxh64_prefix 443 1.1 christos self._rewritten_includes = rewritten_includes 444 1.1 christos self._defs = defs 445 1.1 christos self._replaces = replaces 446 1.1 christos self._undefs = undefs 447 1.1 christos self._excludes = excludes 448 1.1 christos self._seds = seds 449 1.1 christos self._spdx = spdx 450 1.1 christos 451 1.1 christos def _dst_lib_file_paths(self): 452 1.1 christos """ 453 1.1 christos Yields all the file paths in the dst_lib. 454 1.1 christos """ 455 1.1 christos for root, dirname, filenames in os.walk(self._dst_lib): 456 1.1 christos for filename in filenames: 457 1.1 christos filepath = os.path.join(root, filename) 458 1.1 christos yield filepath 459 1.1 christos 460 1.1 christos def _log(self, *args, **kwargs): 461 1.1 christos print(*args, **kwargs) 462 1.1 christos 463 1.1 christos def _copy_file(self, lib_path): 464 1.1 christos suffixes = [".c", ".h", ".S"] 465 1.1 christos if not any((lib_path.endswith(suffix) for suffix in suffixes)): 466 1.1 christos return 467 1.1 christos if lib_path in SKIPPED_FILES: 468 1.1 christos self._log(f"\tSkipping file: {lib_path}") 469 1.1 christos return 470 1.1 christos if self._external_xxhash and lib_path in XXHASH_FILES: 471 1.1 christos self._log(f"\tSkipping xxhash file: {lib_path}") 472 1.1 christos return 473 1.1 christos 474 1.1 christos src_path = os.path.join(self._src_lib, lib_path) 475 1.1 christos dst_path = os.path.join(self._dst_lib, lib_path) 476 1.1 christos self._log(f"\tCopying: {src_path} -> {dst_path}") 477 1.1 christos shutil.copyfile(src_path, dst_path) 478 1.1 christos 479 1.1 christos def _copy_source_lib(self): 480 1.1 christos self._log("Copying source library into output library") 481 1.1 christos 482 1.1 christos assert os.path.exists(self._src_lib) 483 1.1 christos os.makedirs(self._dst_lib, exist_ok=True) 484 1.1 christos self._copy_file("zstd.h") 485 1.1 christos self._copy_file("zstd_errors.h") 486 1.1 christos for subdir in INCLUDED_SUBDIRS: 487 1.1 christos src_dir = os.path.join(self._src_lib, subdir) 488 1.1 christos dst_dir = os.path.join(self._dst_lib, subdir) 489 1.1 christos 490 1.1 christos assert os.path.exists(src_dir) 491 1.1 christos os.makedirs(dst_dir, exist_ok=True) 492 1.1 christos 493 1.1 christos for filename in os.listdir(src_dir): 494 1.1 christos lib_path = os.path.join(subdir, filename) 495 1.1 christos self._copy_file(lib_path) 496 1.1 christos 497 1.1 christos def _copy_zstd_deps(self): 498 1.1 christos dst_zstd_deps = os.path.join(self._dst_lib, "common", "zstd_deps.h") 499 1.1 christos self._log(f"Copying zstd_deps: {self._zstd_deps} -> {dst_zstd_deps}") 500 1.1 christos shutil.copyfile(self._zstd_deps, dst_zstd_deps) 501 1.1 christos 502 1.1 christos def _copy_mem(self): 503 1.1 christos dst_mem = os.path.join(self._dst_lib, "common", "mem.h") 504 1.1 christos self._log(f"Copying mem: {self._mem} -> {dst_mem}") 505 1.1 christos shutil.copyfile(self._mem, dst_mem) 506 1.1 christos 507 1.1 christos def _hardwire_preprocessor(self, name: str, value: Optional[str] = None, undef=False): 508 1.1 christos """ 509 1.1 christos If value=None then hardwire that it is defined, but not what the value is. 510 1.1 christos If undef=True then value must be None. 511 1.1 christos If value='' then the macro is defined to '' exactly. 512 1.1 christos """ 513 1.1 christos assert not (undef and value is not None) 514 1.1 christos for filepath in self._dst_lib_file_paths(): 515 1.1 christos file = FileLines(filepath) 516 1.1 christos 517 1.1 christos def _hardwire_defines(self): 518 1.1 christos self._log("Hardwiring macros") 519 1.1 christos partial_preprocessor = PartialPreprocessor(self._defs, self._replaces, self._undefs) 520 1.1 christos for filepath in self._dst_lib_file_paths(): 521 1.1 christos partial_preprocessor.preprocess(filepath) 522 1.1 christos 523 1.1 christos def _remove_excludes(self): 524 1.1 christos self._log("Removing excluded sections") 525 1.1 christos for exclude in self._excludes: 526 1.1 christos self._log(f"\tRemoving excluded sections for: {exclude}") 527 1.1 christos begin_re = re.compile(f"BEGIN {exclude}") 528 1.1 christos end_re = re.compile(f"END {exclude}") 529 1.1 christos for filepath in self._dst_lib_file_paths(): 530 1.1 christos file = FileLines(filepath) 531 1.1 christos outlines = [] 532 1.1 christos skipped = [] 533 1.1 christos emit = True 534 1.1 christos for line in file.lines: 535 1.1 christos if emit and begin_re.search(line) is not None: 536 1.1 christos assert end_re.search(line) is None 537 1.1 christos emit = False 538 1.1 christos if emit: 539 1.1 christos outlines.append(line) 540 1.1 christos else: 541 1.1 christos skipped.append(line) 542 1.1 christos if end_re.search(line) is not None: 543 1.1 christos assert begin_re.search(line) is None 544 1.1 christos self._log(f"\t\tRemoving excluded section: {exclude}") 545 1.1 christos for s in skipped: 546 1.1 christos self._log(f"\t\t\t- {s}") 547 1.1 christos emit = True 548 1.1 christos skipped = [] 549 1.1 christos if not emit: 550 1.1 christos raise RuntimeError("Excluded section unfinished!") 551 1.1 christos file.lines = outlines 552 1.1 christos file.write() 553 1.1 christos 554 1.1 christos def _rewrite_include(self, original, rewritten): 555 1.1 christos self._log(f"\tRewriting include: {original} -> {rewritten}") 556 1.1 christos regex = re.compile(f"\\s*#\\s*include\\s*(?P<include>{original})") 557 1.1 christos for filepath in self._dst_lib_file_paths(): 558 1.1 christos file = FileLines(filepath) 559 1.1 christos for i, line in enumerate(file.lines): 560 1.1 christos match = regex.match(line) 561 1.1 christos if match is None: 562 1.1 christos continue 563 1.1 christos s = match.start('include') 564 1.1 christos e = match.end('include') 565 1.1 christos file.lines[i] = line[:s] + rewritten + line[e:] 566 1.1 christos file.write() 567 1.1 christos 568 1.1 christos def _rewrite_includes(self): 569 1.1 christos self._log("Rewriting includes") 570 1.1 christos for original, rewritten in self._rewritten_includes: 571 1.1 christos self._rewrite_include(original, rewritten) 572 1.1 christos 573 1.1 christos def _replace_xxh64_prefix(self): 574 1.1 christos if self._xxh64_prefix is None: 575 1.1 christos return 576 1.1 christos self._log(f"Replacing XXH64 prefix with {self._xxh64_prefix}") 577 1.1 christos replacements = [] 578 1.1 christos if self._xxh64_state is not None: 579 1.1 christos replacements.append( 580 1.1 christos (re.compile(r"([^\w]|^)(?P<orig>XXH64_state_t)([^\w]|$)"), self._xxh64_state) 581 1.1 christos ) 582 1.1 christos if self._xxh64_prefix is not None: 583 1.1 christos replacements.append( 584 1.1 christos (re.compile(r"([^\w]|^)(?P<orig>XXH64)[\(_]"), self._xxh64_prefix) 585 1.1 christos ) 586 1.1 christos for filepath in self._dst_lib_file_paths(): 587 1.1 christos file = FileLines(filepath) 588 1.1 christos for i, line in enumerate(file.lines): 589 1.1 christos modified = False 590 1.1 christos for regex, replacement in replacements: 591 1.1 christos match = regex.search(line) 592 1.1 christos while match is not None: 593 1.1 christos modified = True 594 1.1 christos b = match.start('orig') 595 1.1 christos e = match.end('orig') 596 1.1 christos line = line[:b] + replacement + line[e:] 597 1.1 christos match = regex.search(line) 598 1.1 christos if modified: 599 1.1 christos self._log(f"\t- {file.lines[i][:-1]}") 600 1.1 christos self._log(f"\t+ {line[:-1]}") 601 1.1 christos file.lines[i] = line 602 1.1 christos file.write() 603 1.1 christos 604 1.1 christos def _parse_sed(self, sed): 605 1.1 christos assert sed[0] == 's' 606 1.1 christos delim = sed[1] 607 1.1 christos match = re.fullmatch(f's{delim}(.+){delim}(.*){delim}(.*)', sed) 608 1.1 christos assert match is not None 609 1.1 christos regex = re.compile(match.group(1)) 610 1.1 christos format_str = match.group(2) 611 1.1 christos is_global = match.group(3) == 'g' 612 1.1 christos return regex, format_str, is_global 613 1.1 christos 614 1.1 christos def _process_sed(self, sed): 615 1.1 christos self._log(f"Processing sed: {sed}") 616 1.1 christos regex, format_str, is_global = self._parse_sed(sed) 617 1.1 christos 618 1.1 christos for filepath in self._dst_lib_file_paths(): 619 1.1 christos file = FileLines(filepath) 620 1.1 christos for i, line in enumerate(file.lines): 621 1.1 christos modified = False 622 1.1 christos while True: 623 1.1 christos match = regex.search(line) 624 1.1 christos if match is None: 625 1.1 christos break 626 1.1 christos replacement = format_str.format(match.groups(''), match.groupdict('')) 627 1.1 christos b = match.start() 628 1.1 christos e = match.end() 629 1.1 christos line = line[:b] + replacement + line[e:] 630 1.1 christos modified = True 631 1.1 christos if not is_global: 632 1.1 christos break 633 1.1 christos if modified: 634 1.1 christos self._log(f"\t- {file.lines[i][:-1]}") 635 1.1 christos self._log(f"\t+ {line[:-1]}") 636 1.1 christos file.lines[i] = line 637 1.1 christos file.write() 638 1.1 christos 639 1.1 christos def _process_seds(self): 640 1.1 christos self._log("Processing seds") 641 1.1 christos for sed in self._seds: 642 1.1 christos self._process_sed(sed) 643 1.1 christos 644 1.1 christos def _process_spdx(self): 645 1.1 christos if not self._spdx: 646 1.1 christos return 647 1.1 christos self._log("Processing spdx") 648 1.1 christos SPDX_C = "// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause\n" 649 1.1 christos SPDX_H_S = "/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */\n" 650 1.1 christos for filepath in self._dst_lib_file_paths(): 651 1.1 christos file = FileLines(filepath) 652 1.1 christos if file.lines[0] == SPDX_C or file.lines[0] == SPDX_H_S: 653 1.1 christos continue 654 1.1 christos for line in file.lines: 655 1.1 christos if "SPDX-License-Identifier" in line: 656 1.1 christos raise RuntimeError(f"Unexpected SPDX license identifier: {file.filename} {repr(line)}") 657 1.1 christos if file.filename.endswith(".c"): 658 1.1 christos file.lines.insert(0, SPDX_C) 659 1.1 christos elif file.filename.endswith(".h") or file.filename.endswith(".S"): 660 1.1 christos file.lines.insert(0, SPDX_H_S) 661 1.1 christos else: 662 1.1 christos raise RuntimeError(f"Unexpected file extension: {file.filename}") 663 1.1 christos file.write() 664 1.1 christos 665 1.1 christos 666 1.1 christos 667 1.1 christos def go(self): 668 1.1 christos self._copy_source_lib() 669 1.1 christos self._copy_zstd_deps() 670 1.1 christos self._copy_mem() 671 1.1 christos self._hardwire_defines() 672 1.1 christos self._remove_excludes() 673 1.1 christos self._rewrite_includes() 674 1.1 christos self._replace_xxh64_prefix() 675 1.1 christos self._process_seds() 676 1.1 christos self._process_spdx() 677 1.1 christos 678 1.1 christos 679 1.1 christos def parse_optional_pair(defines: [str]) -> [(str, Optional[str])]: 680 1.1 christos output = [] 681 1.1 christos for define in defines: 682 1.1 christos parsed = define.split('=') 683 1.1 christos if len(parsed) == 1: 684 1.1 christos output.append((parsed[0], None)) 685 1.1 christos elif len(parsed) == 2: 686 1.1 christos output.append((parsed[0], parsed[1])) 687 1.1 christos else: 688 1.1 christos raise RuntimeError(f"Bad define: {define}") 689 1.1 christos return output 690 1.1 christos 691 1.1 christos 692 1.1 christos def parse_pair(rewritten_includes: [str]) -> [(str, str)]: 693 1.1 christos output = [] 694 1.1 christos for rewritten_include in rewritten_includes: 695 1.1 christos parsed = rewritten_include.split('=') 696 1.1 christos if len(parsed) == 2: 697 1.1 christos output.append((parsed[0], parsed[1])) 698 1.1 christos else: 699 1.1 christos raise RuntimeError(f"Bad rewritten include: {rewritten_include}") 700 1.1 christos return output 701 1.1 christos 702 1.1 christos 703 1.1 christos 704 1.1 christos def main(name, args): 705 1.1 christos parser = argparse.ArgumentParser(prog=name) 706 1.1 christos parser.add_argument("--zstd-deps", default="zstd_deps.h", help="Zstd dependencies file") 707 1.1 christos parser.add_argument("--mem", default="mem.h", help="Memory module") 708 1.1 christos parser.add_argument("--source-lib", default="../../lib", help="Location of the zstd library") 709 1.1 christos parser.add_argument("--output-lib", default="./freestanding_lib", help="Where to output the freestanding zstd library") 710 1.1 christos parser.add_argument("--xxhash", default=None, help="Alternate external xxhash include e.g. --xxhash='<xxhash.h>'. If set xxhash is not included.") 711 1.1 christos parser.add_argument("--xxh64-state", default=None, help="Alternate XXH64 state type (excluding _) e.g. --xxh64-state='struct xxh64_state'") 712 1.1 christos parser.add_argument("--xxh64-prefix", default=None, help="Alternate XXH64 function prefix (excluding _) e.g. --xxh64-prefix=xxh64") 713 1.1 christos parser.add_argument("--rewrite-include", default=[], dest="rewritten_includes", action="append", help="Rewrite an include REGEX=NEW (e.g. '<stddef\\.h>=<linux/types.h>')") 714 1.1 christos parser.add_argument("--sed", default=[], dest="seds", action="append", help="Apply a sed replacement. Format: `s/REGEX/FORMAT/[g]`. REGEX is a Python regex. FORMAT is a Python format string formatted by the regex dict.") 715 1.1 christos parser.add_argument("--spdx", action="store_true", help="Add SPDX License Identifiers") 716 1.1 christos parser.add_argument("-D", "--define", default=[], dest="defs", action="append", help="Pre-define this macro (can be passed multiple times)") 717 1.1 christos parser.add_argument("-U", "--undefine", default=[], dest="undefs", action="append", help="Pre-undefine this macro (can be passed multiple times)") 718 1.1 christos parser.add_argument("-R", "--replace", default=[], dest="replaces", action="append", help="Pre-define this macro and replace the first ifndef block with its definition") 719 1.1 christos parser.add_argument("-E", "--exclude", default=[], dest="excludes", action="append", help="Exclude all lines between 'BEGIN <EXCLUDE>' and 'END <EXCLUDE>'") 720 1.1 christos args = parser.parse_args(args) 721 1.1 christos 722 1.1 christos # Always remove threading 723 1.1 christos if "ZSTD_MULTITHREAD" not in args.undefs: 724 1.1 christos args.undefs.append("ZSTD_MULTITHREAD") 725 1.1 christos 726 1.1 christos args.defs = parse_optional_pair(args.defs) 727 1.1 christos for name, _ in args.defs: 728 1.1 christos if name in args.undefs: 729 1.1 christos raise RuntimeError(f"{name} is both defined and undefined!") 730 1.1 christos 731 1.1 christos # Always set tracing to 0 732 1.1 christos if "ZSTD_NO_TRACE" not in (arg[0] for arg in args.defs): 733 1.1 christos args.defs.append(("ZSTD_NO_TRACE", None)) 734 1.1 christos args.defs.append(("ZSTD_TRACE", "0")) 735 1.1 christos 736 1.1 christos args.replaces = parse_pair(args.replaces) 737 1.1 christos for name, _ in args.replaces: 738 1.1 christos if name in args.undefs or name in args.defs: 739 1.1 christos raise RuntimeError(f"{name} is both replaced and (un)defined!") 740 1.1 christos 741 1.1 christos args.rewritten_includes = parse_pair(args.rewritten_includes) 742 1.1 christos 743 1.1 christos external_xxhash = False 744 1.1 christos if args.xxhash is not None: 745 1.1 christos external_xxhash = True 746 1.1 christos args.rewritten_includes.append(('"(\\.\\./common/)?xxhash.h"', args.xxhash)) 747 1.1 christos 748 1.1 christos if args.xxh64_prefix is not None: 749 1.1 christos if not external_xxhash: 750 1.1 christos raise RuntimeError("--xxh64-prefix may only be used with --xxhash provided") 751 1.1 christos 752 1.1 christos if args.xxh64_state is not None: 753 1.1 christos if not external_xxhash: 754 1.1 christos raise RuntimeError("--xxh64-state may only be used with --xxhash provided") 755 1.1 christos 756 1.1 christos Freestanding( 757 1.1 christos args.zstd_deps, 758 1.1 christos args.mem, 759 1.1 christos args.source_lib, 760 1.1 christos args.output_lib, 761 1.1 christos external_xxhash, 762 1.1 christos args.xxh64_state, 763 1.1 christos args.xxh64_prefix, 764 1.1 christos args.rewritten_includes, 765 1.1 christos args.defs, 766 1.1 christos args.replaces, 767 1.1 christos args.undefs, 768 1.1 christos args.excludes, 769 1.1 christos args.seds, 770 1.1 christos args.spdx, 771 1.1 christos ).go() 772 1.1 christos 773 1.1 christos if __name__ == "__main__": 774 1.1 christos main(sys.argv[0], sys.argv[1:]) 775