Home | History | Annotate | Line # | Download | only in freestanding_lib
      1  1.1  christos #!/usr/bin/env python3
      2  1.1  christos # ################################################################
      3  1.1  christos # Copyright (c) Meta Platforms, Inc. and affiliates.
      4  1.1  christos # All rights reserved.
      5  1.1  christos #
      6  1.1  christos # This source code is licensed under both the BSD-style license (found in the
      7  1.1  christos # LICENSE file in the root directory of this source tree) and the GPLv2 (found
      8  1.1  christos # in the COPYING file in the root directory of this source tree).
      9  1.1  christos # You may select, at your option, one of the above-listed licenses.
     10  1.1  christos # ##########################################################################
     11  1.1  christos 
     12  1.1  christos import argparse
     13  1.1  christos import contextlib
     14  1.1  christos import os
     15  1.1  christos import re
     16  1.1  christos import shutil
     17  1.1  christos import sys
     18  1.1  christos from typing import Optional
     19  1.1  christos 
     20  1.1  christos 
     21  1.1  christos INCLUDED_SUBDIRS = ["common", "compress", "decompress"]
     22  1.1  christos 
     23  1.1  christos SKIPPED_FILES = [
     24  1.1  christos     "common/mem.h",
     25  1.1  christos     "common/zstd_deps.h",
     26  1.1  christos     "common/pool.c",
     27  1.1  christos     "common/pool.h",
     28  1.1  christos     "common/threading.c",
     29  1.1  christos     "common/threading.h",
     30  1.1  christos     "common/zstd_trace.h",
     31  1.1  christos     "compress/zstdmt_compress.h",
     32  1.1  christos     "compress/zstdmt_compress.c",
     33  1.1  christos ]
     34  1.1  christos 
     35  1.1  christos XXHASH_FILES = [
     36  1.1  christos     "common/xxhash.c",
     37  1.1  christos     "common/xxhash.h",
     38  1.1  christos ]
     39  1.1  christos 
     40  1.1  christos 
     41  1.1  christos class FileLines(object):
     42  1.1  christos     def __init__(self, filename):
     43  1.1  christos         self.filename = filename
     44  1.1  christos         with open(self.filename, "r") as f:
     45  1.1  christos             self.lines = f.readlines()
     46  1.1  christos 
     47  1.1  christos     def write(self):
     48  1.1  christos         with open(self.filename, "w") as f:
     49  1.1  christos             f.write("".join(self.lines))
     50  1.1  christos 
     51  1.1  christos 
     52  1.1  christos class PartialPreprocessor(object):
     53  1.1  christos     """
     54  1.1  christos     Looks for simple ifdefs and ifndefs and replaces them.
     55  1.1  christos     Handles && and ||.
     56  1.1  christos     Has fancy logic to handle translating elifs to ifs.
     57  1.1  christos     Only looks for macros in the first part of the expression with no
     58  1.1  christos     parens.
     59  1.1  christos     Does not handle multi-line macros (only looks in first line).
     60  1.1  christos     """
     61  1.1  christos     def __init__(self, defs: [(str, Optional[str])], replaces: [(str, str)], undefs: [str]):
     62  1.1  christos         MACRO_GROUP = r"(?P<macro>[a-zA-Z_][a-zA-Z_0-9]*)"
     63  1.1  christos         ELIF_GROUP = r"(?P<elif>el)?"
     64  1.1  christos         OP_GROUP = r"(?P<op>&&|\|\|)?"
     65  1.1  christos 
     66  1.1  christos         self._defs = {macro:value for macro, value in defs}
     67  1.1  christos         self._replaces = {macro:value for macro, value in replaces}
     68  1.1  christos         self._defs.update(self._replaces)
     69  1.1  christos         self._undefs = set(undefs)
     70  1.1  christos 
     71  1.1  christos         self._define = re.compile(r"\s*#\s*define")
     72  1.1  christos         self._if = re.compile(r"\s*#\s*if")
     73  1.1  christos         self._elif = re.compile(r"\s*#\s*(?P<elif>el)if")
     74  1.1  christos         self._else = re.compile(r"\s*#\s*(?P<else>else)")
     75  1.1  christos         self._endif = re.compile(r"\s*#\s*endif")
     76  1.1  christos 
     77  1.1  christos         self._ifdef = re.compile(fr"\s*#\s*if(?P<not>n)?def {MACRO_GROUP}\s*")
     78  1.1  christos         self._if_defined = re.compile(
     79  1.1  christos             fr"\s*#\s*{ELIF_GROUP}if\s+(?P<not>!)?\s*defined\s*\(\s*{MACRO_GROUP}\s*\)\s*{OP_GROUP}"
     80  1.1  christos         )
     81  1.1  christos         self._if_defined_value = re.compile(
     82  1.1  christos             fr"\s*#\s*{ELIF_GROUP}if\s+defined\s*\(\s*{MACRO_GROUP}\s*\)\s*"
     83  1.1  christos             fr"(?P<op>&&)\s*"
     84  1.1  christos             fr"(?P<openp>\()?\s*"
     85  1.1  christos             fr"(?P<macro2>[a-zA-Z_][a-zA-Z_0-9]*)\s*"
     86  1.1  christos             fr"(?P<cmp>[=><!]+)\s*"
     87  1.1  christos             fr"(?P<value>[0-9]*)\s*"
     88  1.1  christos             fr"(?P<closep>\))?\s*"
     89  1.1  christos         )
     90  1.1  christos         self._if_true = re.compile(
     91  1.1  christos             fr"\s*#\s*{ELIF_GROUP}if\s+{MACRO_GROUP}\s*{OP_GROUP}"
     92  1.1  christos         )
     93  1.1  christos 
     94  1.1  christos         self._c_comment = re.compile(r"/\*.*?\*/")
     95  1.1  christos         self._cpp_comment = re.compile(r"//")
     96  1.1  christos 
     97  1.1  christos     def _log(self, *args, **kwargs):
     98  1.1  christos         print(*args, **kwargs)
     99  1.1  christos 
    100  1.1  christos     def _strip_comments(self, line):
    101  1.1  christos         # First strip c-style comments (may include //)
    102  1.1  christos         while True:
    103  1.1  christos             m = self._c_comment.search(line)
    104  1.1  christos             if m is None:
    105  1.1  christos                 break
    106  1.1  christos             line = line[:m.start()] + line[m.end():]
    107  1.1  christos 
    108  1.1  christos         # Then strip cpp-style comments
    109  1.1  christos         m = self._cpp_comment.search(line)
    110  1.1  christos         if m is not None:
    111  1.1  christos             line = line[:m.start()]
    112  1.1  christos 
    113  1.1  christos         return line
    114  1.1  christos 
    115  1.1  christos     def _fixup_indentation(self, macro, replace: [str]):
    116  1.1  christos         if len(replace) == 0:
    117  1.1  christos             return replace
    118  1.1  christos         if len(replace) == 1 and self._define.match(replace[0]) is None:
    119  1.1  christos             # If there is only one line, only replace defines
    120  1.1  christos             return replace
    121  1.1  christos 
    122  1.1  christos 
    123  1.1  christos         all_pound = True
    124  1.1  christos         for line in replace:
    125  1.1  christos             if not line.startswith('#'):
    126  1.1  christos                 all_pound = False
    127  1.1  christos         if all_pound:
    128  1.1  christos             replace = [line[1:] for line in replace]
    129  1.1  christos 
    130  1.1  christos         min_spaces = len(replace[0])
    131  1.1  christos         for line in replace:
    132  1.1  christos             spaces = 0
    133  1.1  christos             for i, c in enumerate(line):
    134  1.1  christos                 if c != ' ':
    135  1.1  christos                     # Non-preprocessor line ==> skip the fixup
    136  1.1  christos                     if not all_pound and c != '#':
    137  1.1  christos                         return replace
    138  1.1  christos                     spaces = i
    139  1.1  christos                     break
    140  1.1  christos             min_spaces = min(min_spaces, spaces)
    141  1.1  christos 
    142  1.1  christos         replace = [line[min_spaces:] for line in replace]
    143  1.1  christos 
    144  1.1  christos         if all_pound:
    145  1.1  christos             replace = ["#" + line for line in replace]
    146  1.1  christos 
    147  1.1  christos         return replace
    148  1.1  christos 
    149  1.1  christos     def _handle_if_block(self, macro, idx, is_true, prepend):
    150  1.1  christos         """
    151  1.1  christos         Remove the #if or #elif block starting on this line.
    152  1.1  christos         """
    153  1.1  christos         REMOVE_ONE = 0
    154  1.1  christos         KEEP_ONE = 1
    155  1.1  christos         REMOVE_REST = 2
    156  1.1  christos 
    157  1.1  christos         if is_true:
    158  1.1  christos             state = KEEP_ONE
    159  1.1  christos         else:
    160  1.1  christos             state = REMOVE_ONE
    161  1.1  christos 
    162  1.1  christos         line = self._inlines[idx]
    163  1.1  christos         is_if = self._if.match(line) is not None
    164  1.1  christos         assert is_if or self._elif.match(line) is not None
    165  1.1  christos         depth = 0
    166  1.1  christos 
    167  1.1  christos         start_idx = idx
    168  1.1  christos 
    169  1.1  christos         idx += 1
    170  1.1  christos         replace = prepend
    171  1.1  christos         finished = False
    172  1.1  christos         while idx < len(self._inlines):
    173  1.1  christos             line = self._inlines[idx]
    174  1.1  christos             # Nested if statement
    175  1.1  christos             if self._if.match(line):
    176  1.1  christos                 depth += 1
    177  1.1  christos                 idx += 1
    178  1.1  christos                 continue
    179  1.1  christos             # We're inside a nested statement
    180  1.1  christos             if depth > 0:
    181  1.1  christos                 if self._endif.match(line):
    182  1.1  christos                     depth -= 1
    183  1.1  christos                 idx += 1
    184  1.1  christos                 continue
    185  1.1  christos 
    186  1.1  christos             # We're at the original depth
    187  1.1  christos 
    188  1.1  christos             # Looking only for an endif.
    189  1.1  christos             # We've found a true statement, but haven't
    190  1.1  christos             # completely elided the if block, so we just
    191  1.1  christos             # remove the remainder.
    192  1.1  christos             if state == REMOVE_REST:
    193  1.1  christos                 if self._endif.match(line):
    194  1.1  christos                     if is_if:
    195  1.1  christos                         # Remove the endif because we took the first if
    196  1.1  christos                         idx += 1
    197  1.1  christos                     finished = True
    198  1.1  christos                     break
    199  1.1  christos                 idx += 1
    200  1.1  christos                 continue
    201  1.1  christos 
    202  1.1  christos             if state == KEEP_ONE:
    203  1.1  christos                 m = self._elif.match(line)
    204  1.1  christos                 if self._endif.match(line):
    205  1.1  christos                     replace += self._inlines[start_idx + 1:idx]
    206  1.1  christos                     idx += 1
    207  1.1  christos                     finished = True
    208  1.1  christos                     break
    209  1.1  christos                 if self._elif.match(line) or self._else.match(line):
    210  1.1  christos                     replace += self._inlines[start_idx + 1:idx]
    211  1.1  christos                     state = REMOVE_REST
    212  1.1  christos                 idx += 1
    213  1.1  christos                 continue
    214  1.1  christos 
    215  1.1  christos             if state == REMOVE_ONE:
    216  1.1  christos                 m = self._elif.match(line)
    217  1.1  christos                 if m is not None:
    218  1.1  christos                     if is_if:
    219  1.1  christos                         idx += 1
    220  1.1  christos                         b = m.start('elif')
    221  1.1  christos                         e = m.end('elif')
    222  1.1  christos                         assert e - b == 2
    223  1.1  christos                         replace.append(line[:b] + line[e:])
    224  1.1  christos                     finished = True
    225  1.1  christos                     break
    226  1.1  christos                 m = self._else.match(line)
    227  1.1  christos                 if m is not None:
    228  1.1  christos                     if is_if:
    229  1.1  christos                         idx += 1
    230  1.1  christos                         while self._endif.match(self._inlines[idx]) is None:
    231  1.1  christos                             replace.append(self._inlines[idx])
    232  1.1  christos                             idx += 1
    233  1.1  christos                         idx += 1
    234  1.1  christos                     finished = True
    235  1.1  christos                     break
    236  1.1  christos                 if self._endif.match(line):
    237  1.1  christos                     if is_if:
    238  1.1  christos                         # Remove the endif because no other elifs
    239  1.1  christos                         idx += 1
    240  1.1  christos                     finished = True
    241  1.1  christos                     break
    242  1.1  christos                 idx += 1
    243  1.1  christos                 continue
    244  1.1  christos         if not finished:
    245  1.1  christos             raise RuntimeError("Unterminated if block!")
    246  1.1  christos 
    247  1.1  christos         replace = self._fixup_indentation(macro, replace)
    248  1.1  christos 
    249  1.1  christos         self._log(f"\tHardwiring {macro}")
    250  1.1  christos         if start_idx > 0:
    251  1.1  christos             self._log(f"\t\t  {self._inlines[start_idx - 1][:-1]}")
    252  1.1  christos         for x in range(start_idx, idx):
    253  1.1  christos             self._log(f"\t\t- {self._inlines[x][:-1]}")
    254  1.1  christos         for line in replace:
    255  1.1  christos             self._log(f"\t\t+ {line[:-1]}")
    256  1.1  christos         if idx < len(self._inlines):
    257  1.1  christos             self._log(f"\t\t  {self._inlines[idx][:-1]}")
    258  1.1  christos 
    259  1.1  christos         return idx, replace
    260  1.1  christos 
    261  1.1  christos     def _preprocess_once(self):
    262  1.1  christos         outlines = []
    263  1.1  christos         idx = 0
    264  1.1  christos         changed = False
    265  1.1  christos         while idx < len(self._inlines):
    266  1.1  christos             line = self._inlines[idx]
    267  1.1  christos             sline = self._strip_comments(line)
    268  1.1  christos             m = self._ifdef.fullmatch(sline)
    269  1.1  christos             if_true = False
    270  1.1  christos             if m is None:
    271  1.1  christos                 m = self._if_defined_value.fullmatch(sline)
    272  1.1  christos             if m is None:
    273  1.1  christos                 m = self._if_defined.match(sline)
    274  1.1  christos             if m is None:
    275  1.1  christos                 m = self._if_true.match(sline)
    276  1.1  christos                 if_true = (m is not None)
    277  1.1  christos             if m is None:
    278  1.1  christos                 outlines.append(line)
    279  1.1  christos                 idx += 1
    280  1.1  christos                 continue
    281  1.1  christos 
    282  1.1  christos             groups = m.groupdict()
    283  1.1  christos             macro = groups['macro']
    284  1.1  christos             op = groups.get('op')
    285  1.1  christos 
    286  1.1  christos             if not (macro in self._defs or macro in self._undefs):
    287  1.1  christos                 outlines.append(line)
    288  1.1  christos                 idx += 1
    289  1.1  christos                 continue
    290  1.1  christos 
    291  1.1  christos             defined = macro in self._defs
    292  1.1  christos 
    293  1.1  christos             # Needed variables set:
    294  1.1  christos             # resolved: Is the statement fully resolved?
    295  1.1  christos             # is_true: If resolved, is the statement true?
    296  1.1  christos             ifdef = False
    297  1.1  christos             if if_true:
    298  1.1  christos                 if not defined:
    299  1.1  christos                     outlines.append(line)
    300  1.1  christos                     idx += 1
    301  1.1  christos                     continue
    302  1.1  christos 
    303  1.1  christos                 defined_value = self._defs[macro]
    304  1.1  christos                 is_int = True
    305  1.1  christos                 try:
    306  1.1  christos                     defined_value = int(defined_value)
    307  1.1  christos                 except TypeError:
    308  1.1  christos                     is_int = False
    309  1.1  christos                 except ValueError:
    310  1.1  christos                     is_int = False
    311  1.1  christos 
    312  1.1  christos                 resolved = is_int
    313  1.1  christos                 is_true = (defined_value != 0)
    314  1.1  christos 
    315  1.1  christos                 if resolved and op is not None:
    316  1.1  christos                     if op == '&&':
    317  1.1  christos                         resolved = not is_true
    318  1.1  christos                     else:
    319  1.1  christos                         assert op == '||'
    320  1.1  christos                         resolved = is_true
    321  1.1  christos 
    322  1.1  christos             else:
    323  1.1  christos                 ifdef = groups.get('not') is None
    324  1.1  christos                 elseif = groups.get('elif') is not None
    325  1.1  christos 
    326  1.1  christos                 macro2 = groups.get('macro2')
    327  1.1  christos                 cmp = groups.get('cmp')
    328  1.1  christos                 value = groups.get('value')
    329  1.1  christos                 openp = groups.get('openp')
    330  1.1  christos                 closep = groups.get('closep')
    331  1.1  christos 
    332  1.1  christos                 is_true = (ifdef == defined)
    333  1.1  christos                 resolved = True
    334  1.1  christos                 if op is not None:
    335  1.1  christos                     if op == '&&':
    336  1.1  christos                         resolved = not is_true
    337  1.1  christos                     else:
    338  1.1  christos                         assert op == '||'
    339  1.1  christos                         resolved = is_true
    340  1.1  christos 
    341  1.1  christos                 if macro2 is not None and not resolved:
    342  1.1  christos                     assert ifdef and defined and op == '&&' and cmp is not None
    343  1.1  christos                     # If the statement is true, but we have a single value check, then
    344  1.1  christos                     # check the value.
    345  1.1  christos                     defined_value = self._defs[macro]
    346  1.1  christos                     are_ints = True
    347  1.1  christos                     try:
    348  1.1  christos                         defined_value = int(defined_value)
    349  1.1  christos                         value = int(value)
    350  1.1  christos                     except TypeError:
    351  1.1  christos                         are_ints = False
    352  1.1  christos                     except ValueError:
    353  1.1  christos                         are_ints = False
    354  1.1  christos                     if (
    355  1.1  christos                             macro == macro2 and
    356  1.1  christos                             ((openp is None) == (closep is None)) and
    357  1.1  christos                             are_ints
    358  1.1  christos                     ):
    359  1.1  christos                         resolved = True
    360  1.1  christos                         if cmp == '<':
    361  1.1  christos                             is_true = defined_value < value
    362  1.1  christos                         elif cmp == '<=':
    363  1.1  christos                             is_true = defined_value <= value
    364  1.1  christos                         elif cmp == '==':
    365  1.1  christos                             is_true = defined_value == value
    366  1.1  christos                         elif cmp == '!=':
    367  1.1  christos                             is_true = defined_value != value
    368  1.1  christos                         elif cmp == '>=':
    369  1.1  christos                             is_true = defined_value >= value
    370  1.1  christos                         elif cmp == '>':
    371  1.1  christos                             is_true = defined_value > value
    372  1.1  christos                         else:
    373  1.1  christos                             resolved = False
    374  1.1  christos 
    375  1.1  christos                 if op is not None and not resolved:
    376  1.1  christos                     # Remove the first op in the line + spaces
    377  1.1  christos                     if op == '&&':
    378  1.1  christos                         opre = op
    379  1.1  christos                     else:
    380  1.1  christos                         assert op == '||'
    381  1.1  christos                         opre = r'\|\|'
    382  1.1  christos                     needle = re.compile(fr"(?P<if>\s*#\s*(el)?if\s+).*?(?P<op>{opre}\s*)")
    383  1.1  christos                     match = needle.match(line)
    384  1.1  christos                     assert match is not None
    385  1.1  christos                     newline = line[:match.end('if')] + line[match.end('op'):]
    386  1.1  christos 
    387  1.1  christos                     self._log(f"\tHardwiring partially resolved {macro}")
    388  1.1  christos                     self._log(f"\t\t- {line[:-1]}")
    389  1.1  christos                     self._log(f"\t\t+ {newline[:-1]}")
    390  1.1  christos 
    391  1.1  christos                     outlines.append(newline)
    392  1.1  christos                     idx += 1
    393  1.1  christos                     continue
    394  1.1  christos 
    395  1.1  christos             # Skip any statements we cannot fully compute
    396  1.1  christos             if not resolved:
    397  1.1  christos                 outlines.append(line)
    398  1.1  christos                 idx += 1
    399  1.1  christos                 continue
    400  1.1  christos 
    401  1.1  christos             prepend = []
    402  1.1  christos             if macro in self._replaces:
    403  1.1  christos                 assert not ifdef
    404  1.1  christos                 assert op is None
    405  1.1  christos                 value = self._replaces.pop(macro)
    406  1.1  christos                 prepend = [f"#define {macro} {value}\n"]
    407  1.1  christos 
    408  1.1  christos             idx, replace = self._handle_if_block(macro, idx, is_true, prepend)
    409  1.1  christos             outlines += replace
    410  1.1  christos             changed = True
    411  1.1  christos 
    412  1.1  christos         return changed, outlines
    413  1.1  christos 
    414  1.1  christos     def preprocess(self, filename):
    415  1.1  christos         with open(filename, 'r') as f:
    416  1.1  christos             self._inlines = f.readlines()
    417  1.1  christos         changed = True
    418  1.1  christos         iters = 0
    419  1.1  christos         while changed:
    420  1.1  christos             iters += 1
    421  1.1  christos             changed, outlines = self._preprocess_once()
    422  1.1  christos             self._inlines = outlines
    423  1.1  christos 
    424  1.1  christos         with open(filename, 'w') as f:
    425  1.1  christos             f.write(''.join(self._inlines))
    426  1.1  christos 
    427  1.1  christos 
    428  1.1  christos class Freestanding(object):
    429  1.1  christos     def __init__(
    430  1.1  christos             self, zstd_deps: str, mem: str, source_lib: str, output_lib: str,
    431  1.1  christos             external_xxhash: bool, xxh64_state: Optional[str],
    432  1.1  christos             xxh64_prefix: Optional[str], rewritten_includes: [(str, str)],
    433  1.1  christos             defs: [(str, Optional[str])], replaces: [(str, str)],
    434  1.1  christos             undefs: [str], excludes: [str], seds: [str], spdx: bool,
    435  1.1  christos     ):
    436  1.1  christos         self._zstd_deps = zstd_deps
    437  1.1  christos         self._mem = mem
    438  1.1  christos         self._src_lib = source_lib
    439  1.1  christos         self._dst_lib = output_lib
    440  1.1  christos         self._external_xxhash = external_xxhash
    441  1.1  christos         self._xxh64_state = xxh64_state
    442  1.1  christos         self._xxh64_prefix = xxh64_prefix
    443  1.1  christos         self._rewritten_includes = rewritten_includes
    444  1.1  christos         self._defs = defs
    445  1.1  christos         self._replaces = replaces
    446  1.1  christos         self._undefs = undefs
    447  1.1  christos         self._excludes = excludes
    448  1.1  christos         self._seds = seds
    449  1.1  christos         self._spdx = spdx
    450  1.1  christos 
    451  1.1  christos     def _dst_lib_file_paths(self):
    452  1.1  christos         """
    453  1.1  christos         Yields all the file paths in the dst_lib.
    454  1.1  christos         """
    455  1.1  christos         for root, dirname, filenames in os.walk(self._dst_lib):
    456  1.1  christos             for filename in filenames:
    457  1.1  christos                 filepath = os.path.join(root, filename)
    458  1.1  christos                 yield filepath
    459  1.1  christos 
    460  1.1  christos     def _log(self, *args, **kwargs):
    461  1.1  christos         print(*args, **kwargs)
    462  1.1  christos 
    463  1.1  christos     def _copy_file(self, lib_path):
    464  1.1  christos         suffixes = [".c", ".h", ".S"]
    465  1.1  christos         if not any((lib_path.endswith(suffix) for suffix in suffixes)):
    466  1.1  christos             return
    467  1.1  christos         if lib_path in SKIPPED_FILES:
    468  1.1  christos             self._log(f"\tSkipping file: {lib_path}")
    469  1.1  christos             return
    470  1.1  christos         if self._external_xxhash and lib_path in XXHASH_FILES:
    471  1.1  christos             self._log(f"\tSkipping xxhash file: {lib_path}")
    472  1.1  christos             return
    473  1.1  christos 
    474  1.1  christos         src_path = os.path.join(self._src_lib, lib_path)
    475  1.1  christos         dst_path = os.path.join(self._dst_lib, lib_path)
    476  1.1  christos         self._log(f"\tCopying: {src_path} -> {dst_path}")
    477  1.1  christos         shutil.copyfile(src_path, dst_path)
    478  1.1  christos 
    479  1.1  christos     def _copy_source_lib(self):
    480  1.1  christos         self._log("Copying source library into output library")
    481  1.1  christos 
    482  1.1  christos         assert os.path.exists(self._src_lib)
    483  1.1  christos         os.makedirs(self._dst_lib, exist_ok=True)
    484  1.1  christos         self._copy_file("zstd.h")
    485  1.1  christos         self._copy_file("zstd_errors.h")
    486  1.1  christos         for subdir in INCLUDED_SUBDIRS:
    487  1.1  christos             src_dir = os.path.join(self._src_lib, subdir)
    488  1.1  christos             dst_dir = os.path.join(self._dst_lib, subdir)
    489  1.1  christos 
    490  1.1  christos             assert os.path.exists(src_dir)
    491  1.1  christos             os.makedirs(dst_dir, exist_ok=True)
    492  1.1  christos 
    493  1.1  christos             for filename in os.listdir(src_dir):
    494  1.1  christos                 lib_path = os.path.join(subdir, filename)
    495  1.1  christos                 self._copy_file(lib_path)
    496  1.1  christos 
    497  1.1  christos     def _copy_zstd_deps(self):
    498  1.1  christos         dst_zstd_deps = os.path.join(self._dst_lib, "common", "zstd_deps.h")
    499  1.1  christos         self._log(f"Copying zstd_deps: {self._zstd_deps} -> {dst_zstd_deps}")
    500  1.1  christos         shutil.copyfile(self._zstd_deps, dst_zstd_deps)
    501  1.1  christos 
    502  1.1  christos     def _copy_mem(self):
    503  1.1  christos         dst_mem = os.path.join(self._dst_lib, "common", "mem.h")
    504  1.1  christos         self._log(f"Copying mem: {self._mem} -> {dst_mem}")
    505  1.1  christos         shutil.copyfile(self._mem, dst_mem)
    506  1.1  christos 
    507  1.1  christos     def _hardwire_preprocessor(self, name: str, value: Optional[str] = None, undef=False):
    508  1.1  christos         """
    509  1.1  christos         If value=None then hardwire that it is defined, but not what the value is.
    510  1.1  christos         If undef=True then value must be None.
    511  1.1  christos         If value='' then the macro is defined to '' exactly.
    512  1.1  christos         """
    513  1.1  christos         assert not (undef and value is not None)
    514  1.1  christos         for filepath in self._dst_lib_file_paths():
    515  1.1  christos             file = FileLines(filepath)
    516  1.1  christos 
    517  1.1  christos     def _hardwire_defines(self):
    518  1.1  christos         self._log("Hardwiring macros")
    519  1.1  christos         partial_preprocessor = PartialPreprocessor(self._defs, self._replaces, self._undefs)
    520  1.1  christos         for filepath in self._dst_lib_file_paths():
    521  1.1  christos             partial_preprocessor.preprocess(filepath)
    522  1.1  christos 
    523  1.1  christos     def _remove_excludes(self):
    524  1.1  christos         self._log("Removing excluded sections")
    525  1.1  christos         for exclude in self._excludes:
    526  1.1  christos             self._log(f"\tRemoving excluded sections for: {exclude}")
    527  1.1  christos             begin_re = re.compile(f"BEGIN {exclude}")
    528  1.1  christos             end_re = re.compile(f"END {exclude}")
    529  1.1  christos             for filepath in self._dst_lib_file_paths():
    530  1.1  christos                 file = FileLines(filepath)
    531  1.1  christos                 outlines = []
    532  1.1  christos                 skipped = []
    533  1.1  christos                 emit = True
    534  1.1  christos                 for line in file.lines:
    535  1.1  christos                     if emit and begin_re.search(line) is not None:
    536  1.1  christos                         assert end_re.search(line) is None
    537  1.1  christos                         emit = False
    538  1.1  christos                     if emit:
    539  1.1  christos                         outlines.append(line)
    540  1.1  christos                     else:
    541  1.1  christos                         skipped.append(line)
    542  1.1  christos                         if end_re.search(line) is not None:
    543  1.1  christos                             assert begin_re.search(line) is None
    544  1.1  christos                             self._log(f"\t\tRemoving excluded section: {exclude}")
    545  1.1  christos                             for s in skipped:
    546  1.1  christos                                 self._log(f"\t\t\t- {s}")
    547  1.1  christos                             emit = True
    548  1.1  christos                             skipped = []
    549  1.1  christos                 if not emit:
    550  1.1  christos                     raise RuntimeError("Excluded section unfinished!")
    551  1.1  christos                 file.lines = outlines
    552  1.1  christos                 file.write()
    553  1.1  christos 
    554  1.1  christos     def _rewrite_include(self, original, rewritten):
    555  1.1  christos         self._log(f"\tRewriting include: {original} -> {rewritten}")
    556  1.1  christos         regex = re.compile(f"\\s*#\\s*include\\s*(?P<include>{original})")
    557  1.1  christos         for filepath in self._dst_lib_file_paths():
    558  1.1  christos             file = FileLines(filepath)
    559  1.1  christos             for i, line in enumerate(file.lines):
    560  1.1  christos                 match = regex.match(line)
    561  1.1  christos                 if match is None:
    562  1.1  christos                     continue
    563  1.1  christos                 s = match.start('include')
    564  1.1  christos                 e = match.end('include')
    565  1.1  christos                 file.lines[i] = line[:s] + rewritten + line[e:]
    566  1.1  christos             file.write()
    567  1.1  christos 
    568  1.1  christos     def _rewrite_includes(self):
    569  1.1  christos         self._log("Rewriting includes")
    570  1.1  christos         for original, rewritten in self._rewritten_includes:
    571  1.1  christos             self._rewrite_include(original, rewritten)
    572  1.1  christos 
    573  1.1  christos     def _replace_xxh64_prefix(self):
    574  1.1  christos         if self._xxh64_prefix is None:
    575  1.1  christos             return
    576  1.1  christos         self._log(f"Replacing XXH64 prefix with {self._xxh64_prefix}")
    577  1.1  christos         replacements = []
    578  1.1  christos         if self._xxh64_state is not None:
    579  1.1  christos             replacements.append(
    580  1.1  christos                 (re.compile(r"([^\w]|^)(?P<orig>XXH64_state_t)([^\w]|$)"), self._xxh64_state)
    581  1.1  christos             )
    582  1.1  christos         if self._xxh64_prefix is not None:
    583  1.1  christos             replacements.append(
    584  1.1  christos                 (re.compile(r"([^\w]|^)(?P<orig>XXH64)[\(_]"), self._xxh64_prefix)
    585  1.1  christos             )
    586  1.1  christos         for filepath in self._dst_lib_file_paths():
    587  1.1  christos             file = FileLines(filepath)
    588  1.1  christos             for i, line in enumerate(file.lines):
    589  1.1  christos                 modified = False
    590  1.1  christos                 for regex, replacement in replacements:
    591  1.1  christos                     match = regex.search(line)
    592  1.1  christos                     while match is not None:
    593  1.1  christos                         modified = True
    594  1.1  christos                         b = match.start('orig')
    595  1.1  christos                         e = match.end('orig')
    596  1.1  christos                         line = line[:b] + replacement + line[e:]
    597  1.1  christos                         match = regex.search(line)
    598  1.1  christos                 if modified:
    599  1.1  christos                     self._log(f"\t- {file.lines[i][:-1]}")
    600  1.1  christos                     self._log(f"\t+ {line[:-1]}")
    601  1.1  christos                 file.lines[i] = line
    602  1.1  christos             file.write()
    603  1.1  christos 
    604  1.1  christos     def _parse_sed(self, sed):
    605  1.1  christos         assert sed[0] == 's'
    606  1.1  christos         delim = sed[1]
    607  1.1  christos         match = re.fullmatch(f's{delim}(.+){delim}(.*){delim}(.*)', sed)
    608  1.1  christos         assert match is not None
    609  1.1  christos         regex = re.compile(match.group(1))
    610  1.1  christos         format_str = match.group(2)
    611  1.1  christos         is_global = match.group(3) == 'g'
    612  1.1  christos         return regex, format_str, is_global
    613  1.1  christos 
    614  1.1  christos     def _process_sed(self, sed):
    615  1.1  christos         self._log(f"Processing sed: {sed}")
    616  1.1  christos         regex, format_str, is_global = self._parse_sed(sed)
    617  1.1  christos 
    618  1.1  christos         for filepath in self._dst_lib_file_paths():
    619  1.1  christos             file = FileLines(filepath)
    620  1.1  christos             for i, line in enumerate(file.lines):
    621  1.1  christos                 modified = False
    622  1.1  christos                 while True:
    623  1.1  christos                     match = regex.search(line)
    624  1.1  christos                     if match is None:
    625  1.1  christos                         break
    626  1.1  christos                     replacement = format_str.format(match.groups(''), match.groupdict(''))
    627  1.1  christos                     b = match.start()
    628  1.1  christos                     e = match.end()
    629  1.1  christos                     line = line[:b] + replacement + line[e:]
    630  1.1  christos                     modified = True
    631  1.1  christos                     if not is_global:
    632  1.1  christos                         break
    633  1.1  christos                 if modified:
    634  1.1  christos                     self._log(f"\t- {file.lines[i][:-1]}")
    635  1.1  christos                     self._log(f"\t+ {line[:-1]}")
    636  1.1  christos                 file.lines[i] = line
    637  1.1  christos             file.write()
    638  1.1  christos 
    639  1.1  christos     def _process_seds(self):
    640  1.1  christos         self._log("Processing seds")
    641  1.1  christos         for sed in self._seds:
    642  1.1  christos             self._process_sed(sed)
    643  1.1  christos 
    644  1.1  christos     def _process_spdx(self):
    645  1.1  christos         if not self._spdx:
    646  1.1  christos             return
    647  1.1  christos         self._log("Processing spdx")
    648  1.1  christos         SPDX_C = "// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause\n"
    649  1.1  christos         SPDX_H_S = "/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */\n"
    650  1.1  christos         for filepath in self._dst_lib_file_paths():
    651  1.1  christos             file = FileLines(filepath)
    652  1.1  christos             if file.lines[0] == SPDX_C or file.lines[0] == SPDX_H_S:
    653  1.1  christos                 continue
    654  1.1  christos             for line in file.lines:
    655  1.1  christos                 if "SPDX-License-Identifier" in line:
    656  1.1  christos                     raise RuntimeError(f"Unexpected SPDX license identifier: {file.filename} {repr(line)}")
    657  1.1  christos             if file.filename.endswith(".c"):
    658  1.1  christos                 file.lines.insert(0, SPDX_C)
    659  1.1  christos             elif file.filename.endswith(".h") or file.filename.endswith(".S"):
    660  1.1  christos                 file.lines.insert(0, SPDX_H_S)
    661  1.1  christos             else:
    662  1.1  christos                 raise RuntimeError(f"Unexpected file extension: {file.filename}")
    663  1.1  christos             file.write()
    664  1.1  christos 
    665  1.1  christos 
    666  1.1  christos 
    667  1.1  christos     def go(self):
    668  1.1  christos         self._copy_source_lib()
    669  1.1  christos         self._copy_zstd_deps()
    670  1.1  christos         self._copy_mem()
    671  1.1  christos         self._hardwire_defines()
    672  1.1  christos         self._remove_excludes()
    673  1.1  christos         self._rewrite_includes()
    674  1.1  christos         self._replace_xxh64_prefix()
    675  1.1  christos         self._process_seds()
    676  1.1  christos         self._process_spdx()
    677  1.1  christos 
    678  1.1  christos 
    679  1.1  christos def parse_optional_pair(defines: [str]) -> [(str, Optional[str])]:
    680  1.1  christos     output = []
    681  1.1  christos     for define in defines:
    682  1.1  christos         parsed = define.split('=')
    683  1.1  christos         if len(parsed) == 1:
    684  1.1  christos             output.append((parsed[0], None))
    685  1.1  christos         elif len(parsed) == 2:
    686  1.1  christos             output.append((parsed[0], parsed[1]))
    687  1.1  christos         else:
    688  1.1  christos             raise RuntimeError(f"Bad define: {define}")
    689  1.1  christos     return output
    690  1.1  christos 
    691  1.1  christos 
    692  1.1  christos def parse_pair(rewritten_includes: [str]) -> [(str, str)]:
    693  1.1  christos     output = []
    694  1.1  christos     for rewritten_include in rewritten_includes:
    695  1.1  christos         parsed = rewritten_include.split('=')
    696  1.1  christos         if len(parsed) == 2:
    697  1.1  christos             output.append((parsed[0], parsed[1]))
    698  1.1  christos         else:
    699  1.1  christos             raise RuntimeError(f"Bad rewritten include: {rewritten_include}")
    700  1.1  christos     return output
    701  1.1  christos 
    702  1.1  christos 
    703  1.1  christos 
    704  1.1  christos def main(name, args):
    705  1.1  christos     parser = argparse.ArgumentParser(prog=name)
    706  1.1  christos     parser.add_argument("--zstd-deps", default="zstd_deps.h", help="Zstd dependencies file")
    707  1.1  christos     parser.add_argument("--mem", default="mem.h", help="Memory module")
    708  1.1  christos     parser.add_argument("--source-lib", default="../../lib", help="Location of the zstd library")
    709  1.1  christos     parser.add_argument("--output-lib", default="./freestanding_lib", help="Where to output the freestanding zstd library")
    710  1.1  christos     parser.add_argument("--xxhash", default=None, help="Alternate external xxhash include e.g. --xxhash='<xxhash.h>'. If set xxhash is not included.")
    711  1.1  christos     parser.add_argument("--xxh64-state", default=None, help="Alternate XXH64 state type (excluding _) e.g. --xxh64-state='struct xxh64_state'")
    712  1.1  christos     parser.add_argument("--xxh64-prefix", default=None, help="Alternate XXH64 function prefix (excluding _) e.g. --xxh64-prefix=xxh64")
    713  1.1  christos     parser.add_argument("--rewrite-include", default=[], dest="rewritten_includes", action="append", help="Rewrite an include REGEX=NEW (e.g. '<stddef\\.h>=<linux/types.h>')")
    714  1.1  christos     parser.add_argument("--sed", default=[], dest="seds", action="append", help="Apply a sed replacement. Format: `s/REGEX/FORMAT/[g]`. REGEX is a Python regex. FORMAT is a Python format string formatted by the regex dict.")
    715  1.1  christos     parser.add_argument("--spdx", action="store_true", help="Add SPDX License Identifiers")
    716  1.1  christos     parser.add_argument("-D", "--define", default=[], dest="defs", action="append", help="Pre-define this macro (can be passed multiple times)")
    717  1.1  christos     parser.add_argument("-U", "--undefine", default=[], dest="undefs", action="append", help="Pre-undefine this macro (can be passed multiple times)")
    718  1.1  christos     parser.add_argument("-R", "--replace", default=[], dest="replaces", action="append", help="Pre-define this macro and replace the first ifndef block with its definition")
    719  1.1  christos     parser.add_argument("-E", "--exclude", default=[], dest="excludes", action="append", help="Exclude all lines between 'BEGIN <EXCLUDE>' and 'END <EXCLUDE>'")
    720  1.1  christos     args = parser.parse_args(args)
    721  1.1  christos 
    722  1.1  christos     # Always remove threading
    723  1.1  christos     if "ZSTD_MULTITHREAD" not in args.undefs:
    724  1.1  christos         args.undefs.append("ZSTD_MULTITHREAD")
    725  1.1  christos 
    726  1.1  christos     args.defs = parse_optional_pair(args.defs)
    727  1.1  christos     for name, _ in args.defs:
    728  1.1  christos         if name in args.undefs:
    729  1.1  christos             raise RuntimeError(f"{name} is both defined and undefined!")
    730  1.1  christos 
    731  1.1  christos     # Always set tracing to 0
    732  1.1  christos     if "ZSTD_NO_TRACE" not in (arg[0] for arg in args.defs):
    733  1.1  christos         args.defs.append(("ZSTD_NO_TRACE", None))
    734  1.1  christos         args.defs.append(("ZSTD_TRACE", "0"))
    735  1.1  christos 
    736  1.1  christos     args.replaces = parse_pair(args.replaces)
    737  1.1  christos     for name, _ in args.replaces:
    738  1.1  christos         if name in args.undefs or name in args.defs:
    739  1.1  christos             raise RuntimeError(f"{name} is both replaced and (un)defined!")
    740  1.1  christos 
    741  1.1  christos     args.rewritten_includes = parse_pair(args.rewritten_includes)
    742  1.1  christos 
    743  1.1  christos     external_xxhash = False
    744  1.1  christos     if args.xxhash is not None:
    745  1.1  christos         external_xxhash = True
    746  1.1  christos         args.rewritten_includes.append(('"(\\.\\./common/)?xxhash.h"', args.xxhash))
    747  1.1  christos 
    748  1.1  christos     if args.xxh64_prefix is not None:
    749  1.1  christos         if not external_xxhash:
    750  1.1  christos             raise RuntimeError("--xxh64-prefix may only be used with --xxhash provided")
    751  1.1  christos 
    752  1.1  christos     if args.xxh64_state is not None:
    753  1.1  christos         if not external_xxhash:
    754  1.1  christos             raise RuntimeError("--xxh64-state may only be used with --xxhash provided")
    755  1.1  christos 
    756  1.1  christos     Freestanding(
    757  1.1  christos         args.zstd_deps,
    758  1.1  christos         args.mem,
    759  1.1  christos         args.source_lib,
    760  1.1  christos         args.output_lib,
    761  1.1  christos         external_xxhash,
    762  1.1  christos         args.xxh64_state,
    763  1.1  christos         args.xxh64_prefix,
    764  1.1  christos         args.rewritten_includes,
    765  1.1  christos         args.defs,
    766  1.1  christos         args.replaces,
    767  1.1  christos         args.undefs,
    768  1.1  christos         args.excludes,
    769  1.1  christos         args.seds,
    770  1.1  christos         args.spdx,
    771  1.1  christos     ).go()
    772  1.1  christos 
    773  1.1  christos if __name__ == "__main__":
    774  1.1  christos     main(sys.argv[0], sys.argv[1:])
    775