freestanding.py revision 1.1 1 1.1 christos #!/usr/bin/env python3
2 1.1 christos # ################################################################
3 1.1 christos # Copyright (c) Meta Platforms, Inc. and affiliates.
4 1.1 christos # All rights reserved.
5 1.1 christos #
6 1.1 christos # This source code is licensed under both the BSD-style license (found in the
7 1.1 christos # LICENSE file in the root directory of this source tree) and the GPLv2 (found
8 1.1 christos # in the COPYING file in the root directory of this source tree).
9 1.1 christos # You may select, at your option, one of the above-listed licenses.
10 1.1 christos # ##########################################################################
11 1.1 christos
12 1.1 christos import argparse
13 1.1 christos import contextlib
14 1.1 christos import os
15 1.1 christos import re
16 1.1 christos import shutil
17 1.1 christos import sys
18 1.1 christos from typing import Optional
19 1.1 christos
20 1.1 christos
21 1.1 christos INCLUDED_SUBDIRS = ["common", "compress", "decompress"]
22 1.1 christos
23 1.1 christos SKIPPED_FILES = [
24 1.1 christos "common/mem.h",
25 1.1 christos "common/zstd_deps.h",
26 1.1 christos "common/pool.c",
27 1.1 christos "common/pool.h",
28 1.1 christos "common/threading.c",
29 1.1 christos "common/threading.h",
30 1.1 christos "common/zstd_trace.h",
31 1.1 christos "compress/zstdmt_compress.h",
32 1.1 christos "compress/zstdmt_compress.c",
33 1.1 christos ]
34 1.1 christos
35 1.1 christos XXHASH_FILES = [
36 1.1 christos "common/xxhash.c",
37 1.1 christos "common/xxhash.h",
38 1.1 christos ]
39 1.1 christos
40 1.1 christos
41 1.1 christos class FileLines(object):
42 1.1 christos def __init__(self, filename):
43 1.1 christos self.filename = filename
44 1.1 christos with open(self.filename, "r") as f:
45 1.1 christos self.lines = f.readlines()
46 1.1 christos
47 1.1 christos def write(self):
48 1.1 christos with open(self.filename, "w") as f:
49 1.1 christos f.write("".join(self.lines))
50 1.1 christos
51 1.1 christos
52 1.1 christos class PartialPreprocessor(object):
53 1.1 christos """
54 1.1 christos Looks for simple ifdefs and ifndefs and replaces them.
55 1.1 christos Handles && and ||.
56 1.1 christos Has fancy logic to handle translating elifs to ifs.
57 1.1 christos Only looks for macros in the first part of the expression with no
58 1.1 christos parens.
59 1.1 christos Does not handle multi-line macros (only looks in first line).
60 1.1 christos """
61 1.1 christos def __init__(self, defs: [(str, Optional[str])], replaces: [(str, str)], undefs: [str]):
62 1.1 christos MACRO_GROUP = r"(?P<macro>[a-zA-Z_][a-zA-Z_0-9]*)"
63 1.1 christos ELIF_GROUP = r"(?P<elif>el)?"
64 1.1 christos OP_GROUP = r"(?P<op>&&|\|\|)?"
65 1.1 christos
66 1.1 christos self._defs = {macro:value for macro, value in defs}
67 1.1 christos self._replaces = {macro:value for macro, value in replaces}
68 1.1 christos self._defs.update(self._replaces)
69 1.1 christos self._undefs = set(undefs)
70 1.1 christos
71 1.1 christos self._define = re.compile(r"\s*#\s*define")
72 1.1 christos self._if = re.compile(r"\s*#\s*if")
73 1.1 christos self._elif = re.compile(r"\s*#\s*(?P<elif>el)if")
74 1.1 christos self._else = re.compile(r"\s*#\s*(?P<else>else)")
75 1.1 christos self._endif = re.compile(r"\s*#\s*endif")
76 1.1 christos
77 1.1 christos self._ifdef = re.compile(fr"\s*#\s*if(?P<not>n)?def {MACRO_GROUP}\s*")
78 1.1 christos self._if_defined = re.compile(
79 1.1 christos fr"\s*#\s*{ELIF_GROUP}if\s+(?P<not>!)?\s*defined\s*\(\s*{MACRO_GROUP}\s*\)\s*{OP_GROUP}"
80 1.1 christos )
81 1.1 christos self._if_defined_value = re.compile(
82 1.1 christos fr"\s*#\s*{ELIF_GROUP}if\s+defined\s*\(\s*{MACRO_GROUP}\s*\)\s*"
83 1.1 christos fr"(?P<op>&&)\s*"
84 1.1 christos fr"(?P<openp>\()?\s*"
85 1.1 christos fr"(?P<macro2>[a-zA-Z_][a-zA-Z_0-9]*)\s*"
86 1.1 christos fr"(?P<cmp>[=><!]+)\s*"
87 1.1 christos fr"(?P<value>[0-9]*)\s*"
88 1.1 christos fr"(?P<closep>\))?\s*"
89 1.1 christos )
90 1.1 christos self._if_true = re.compile(
91 1.1 christos fr"\s*#\s*{ELIF_GROUP}if\s+{MACRO_GROUP}\s*{OP_GROUP}"
92 1.1 christos )
93 1.1 christos
94 1.1 christos self._c_comment = re.compile(r"/\*.*?\*/")
95 1.1 christos self._cpp_comment = re.compile(r"//")
96 1.1 christos
97 1.1 christos def _log(self, *args, **kwargs):
98 1.1 christos print(*args, **kwargs)
99 1.1 christos
100 1.1 christos def _strip_comments(self, line):
101 1.1 christos # First strip c-style comments (may include //)
102 1.1 christos while True:
103 1.1 christos m = self._c_comment.search(line)
104 1.1 christos if m is None:
105 1.1 christos break
106 1.1 christos line = line[:m.start()] + line[m.end():]
107 1.1 christos
108 1.1 christos # Then strip cpp-style comments
109 1.1 christos m = self._cpp_comment.search(line)
110 1.1 christos if m is not None:
111 1.1 christos line = line[:m.start()]
112 1.1 christos
113 1.1 christos return line
114 1.1 christos
115 1.1 christos def _fixup_indentation(self, macro, replace: [str]):
116 1.1 christos if len(replace) == 0:
117 1.1 christos return replace
118 1.1 christos if len(replace) == 1 and self._define.match(replace[0]) is None:
119 1.1 christos # If there is only one line, only replace defines
120 1.1 christos return replace
121 1.1 christos
122 1.1 christos
123 1.1 christos all_pound = True
124 1.1 christos for line in replace:
125 1.1 christos if not line.startswith('#'):
126 1.1 christos all_pound = False
127 1.1 christos if all_pound:
128 1.1 christos replace = [line[1:] for line in replace]
129 1.1 christos
130 1.1 christos min_spaces = len(replace[0])
131 1.1 christos for line in replace:
132 1.1 christos spaces = 0
133 1.1 christos for i, c in enumerate(line):
134 1.1 christos if c != ' ':
135 1.1 christos # Non-preprocessor line ==> skip the fixup
136 1.1 christos if not all_pound and c != '#':
137 1.1 christos return replace
138 1.1 christos spaces = i
139 1.1 christos break
140 1.1 christos min_spaces = min(min_spaces, spaces)
141 1.1 christos
142 1.1 christos replace = [line[min_spaces:] for line in replace]
143 1.1 christos
144 1.1 christos if all_pound:
145 1.1 christos replace = ["#" + line for line in replace]
146 1.1 christos
147 1.1 christos return replace
148 1.1 christos
149 1.1 christos def _handle_if_block(self, macro, idx, is_true, prepend):
150 1.1 christos """
151 1.1 christos Remove the #if or #elif block starting on this line.
152 1.1 christos """
153 1.1 christos REMOVE_ONE = 0
154 1.1 christos KEEP_ONE = 1
155 1.1 christos REMOVE_REST = 2
156 1.1 christos
157 1.1 christos if is_true:
158 1.1 christos state = KEEP_ONE
159 1.1 christos else:
160 1.1 christos state = REMOVE_ONE
161 1.1 christos
162 1.1 christos line = self._inlines[idx]
163 1.1 christos is_if = self._if.match(line) is not None
164 1.1 christos assert is_if or self._elif.match(line) is not None
165 1.1 christos depth = 0
166 1.1 christos
167 1.1 christos start_idx = idx
168 1.1 christos
169 1.1 christos idx += 1
170 1.1 christos replace = prepend
171 1.1 christos finished = False
172 1.1 christos while idx < len(self._inlines):
173 1.1 christos line = self._inlines[idx]
174 1.1 christos # Nested if statement
175 1.1 christos if self._if.match(line):
176 1.1 christos depth += 1
177 1.1 christos idx += 1
178 1.1 christos continue
179 1.1 christos # We're inside a nested statement
180 1.1 christos if depth > 0:
181 1.1 christos if self._endif.match(line):
182 1.1 christos depth -= 1
183 1.1 christos idx += 1
184 1.1 christos continue
185 1.1 christos
186 1.1 christos # We're at the original depth
187 1.1 christos
188 1.1 christos # Looking only for an endif.
189 1.1 christos # We've found a true statement, but haven't
190 1.1 christos # completely elided the if block, so we just
191 1.1 christos # remove the remainder.
192 1.1 christos if state == REMOVE_REST:
193 1.1 christos if self._endif.match(line):
194 1.1 christos if is_if:
195 1.1 christos # Remove the endif because we took the first if
196 1.1 christos idx += 1
197 1.1 christos finished = True
198 1.1 christos break
199 1.1 christos idx += 1
200 1.1 christos continue
201 1.1 christos
202 1.1 christos if state == KEEP_ONE:
203 1.1 christos m = self._elif.match(line)
204 1.1 christos if self._endif.match(line):
205 1.1 christos replace += self._inlines[start_idx + 1:idx]
206 1.1 christos idx += 1
207 1.1 christos finished = True
208 1.1 christos break
209 1.1 christos if self._elif.match(line) or self._else.match(line):
210 1.1 christos replace += self._inlines[start_idx + 1:idx]
211 1.1 christos state = REMOVE_REST
212 1.1 christos idx += 1
213 1.1 christos continue
214 1.1 christos
215 1.1 christos if state == REMOVE_ONE:
216 1.1 christos m = self._elif.match(line)
217 1.1 christos if m is not None:
218 1.1 christos if is_if:
219 1.1 christos idx += 1
220 1.1 christos b = m.start('elif')
221 1.1 christos e = m.end('elif')
222 1.1 christos assert e - b == 2
223 1.1 christos replace.append(line[:b] + line[e:])
224 1.1 christos finished = True
225 1.1 christos break
226 1.1 christos m = self._else.match(line)
227 1.1 christos if m is not None:
228 1.1 christos if is_if:
229 1.1 christos idx += 1
230 1.1 christos while self._endif.match(self._inlines[idx]) is None:
231 1.1 christos replace.append(self._inlines[idx])
232 1.1 christos idx += 1
233 1.1 christos idx += 1
234 1.1 christos finished = True
235 1.1 christos break
236 1.1 christos if self._endif.match(line):
237 1.1 christos if is_if:
238 1.1 christos # Remove the endif because no other elifs
239 1.1 christos idx += 1
240 1.1 christos finished = True
241 1.1 christos break
242 1.1 christos idx += 1
243 1.1 christos continue
244 1.1 christos if not finished:
245 1.1 christos raise RuntimeError("Unterminated if block!")
246 1.1 christos
247 1.1 christos replace = self._fixup_indentation(macro, replace)
248 1.1 christos
249 1.1 christos self._log(f"\tHardwiring {macro}")
250 1.1 christos if start_idx > 0:
251 1.1 christos self._log(f"\t\t {self._inlines[start_idx - 1][:-1]}")
252 1.1 christos for x in range(start_idx, idx):
253 1.1 christos self._log(f"\t\t- {self._inlines[x][:-1]}")
254 1.1 christos for line in replace:
255 1.1 christos self._log(f"\t\t+ {line[:-1]}")
256 1.1 christos if idx < len(self._inlines):
257 1.1 christos self._log(f"\t\t {self._inlines[idx][:-1]}")
258 1.1 christos
259 1.1 christos return idx, replace
260 1.1 christos
261 1.1 christos def _preprocess_once(self):
262 1.1 christos outlines = []
263 1.1 christos idx = 0
264 1.1 christos changed = False
265 1.1 christos while idx < len(self._inlines):
266 1.1 christos line = self._inlines[idx]
267 1.1 christos sline = self._strip_comments(line)
268 1.1 christos m = self._ifdef.fullmatch(sline)
269 1.1 christos if_true = False
270 1.1 christos if m is None:
271 1.1 christos m = self._if_defined_value.fullmatch(sline)
272 1.1 christos if m is None:
273 1.1 christos m = self._if_defined.match(sline)
274 1.1 christos if m is None:
275 1.1 christos m = self._if_true.match(sline)
276 1.1 christos if_true = (m is not None)
277 1.1 christos if m is None:
278 1.1 christos outlines.append(line)
279 1.1 christos idx += 1
280 1.1 christos continue
281 1.1 christos
282 1.1 christos groups = m.groupdict()
283 1.1 christos macro = groups['macro']
284 1.1 christos op = groups.get('op')
285 1.1 christos
286 1.1 christos if not (macro in self._defs or macro in self._undefs):
287 1.1 christos outlines.append(line)
288 1.1 christos idx += 1
289 1.1 christos continue
290 1.1 christos
291 1.1 christos defined = macro in self._defs
292 1.1 christos
293 1.1 christos # Needed variables set:
294 1.1 christos # resolved: Is the statement fully resolved?
295 1.1 christos # is_true: If resolved, is the statement true?
296 1.1 christos ifdef = False
297 1.1 christos if if_true:
298 1.1 christos if not defined:
299 1.1 christos outlines.append(line)
300 1.1 christos idx += 1
301 1.1 christos continue
302 1.1 christos
303 1.1 christos defined_value = self._defs[macro]
304 1.1 christos is_int = True
305 1.1 christos try:
306 1.1 christos defined_value = int(defined_value)
307 1.1 christos except TypeError:
308 1.1 christos is_int = False
309 1.1 christos except ValueError:
310 1.1 christos is_int = False
311 1.1 christos
312 1.1 christos resolved = is_int
313 1.1 christos is_true = (defined_value != 0)
314 1.1 christos
315 1.1 christos if resolved and op is not None:
316 1.1 christos if op == '&&':
317 1.1 christos resolved = not is_true
318 1.1 christos else:
319 1.1 christos assert op == '||'
320 1.1 christos resolved = is_true
321 1.1 christos
322 1.1 christos else:
323 1.1 christos ifdef = groups.get('not') is None
324 1.1 christos elseif = groups.get('elif') is not None
325 1.1 christos
326 1.1 christos macro2 = groups.get('macro2')
327 1.1 christos cmp = groups.get('cmp')
328 1.1 christos value = groups.get('value')
329 1.1 christos openp = groups.get('openp')
330 1.1 christos closep = groups.get('closep')
331 1.1 christos
332 1.1 christos is_true = (ifdef == defined)
333 1.1 christos resolved = True
334 1.1 christos if op is not None:
335 1.1 christos if op == '&&':
336 1.1 christos resolved = not is_true
337 1.1 christos else:
338 1.1 christos assert op == '||'
339 1.1 christos resolved = is_true
340 1.1 christos
341 1.1 christos if macro2 is not None and not resolved:
342 1.1 christos assert ifdef and defined and op == '&&' and cmp is not None
343 1.1 christos # If the statement is true, but we have a single value check, then
344 1.1 christos # check the value.
345 1.1 christos defined_value = self._defs[macro]
346 1.1 christos are_ints = True
347 1.1 christos try:
348 1.1 christos defined_value = int(defined_value)
349 1.1 christos value = int(value)
350 1.1 christos except TypeError:
351 1.1 christos are_ints = False
352 1.1 christos except ValueError:
353 1.1 christos are_ints = False
354 1.1 christos if (
355 1.1 christos macro == macro2 and
356 1.1 christos ((openp is None) == (closep is None)) and
357 1.1 christos are_ints
358 1.1 christos ):
359 1.1 christos resolved = True
360 1.1 christos if cmp == '<':
361 1.1 christos is_true = defined_value < value
362 1.1 christos elif cmp == '<=':
363 1.1 christos is_true = defined_value <= value
364 1.1 christos elif cmp == '==':
365 1.1 christos is_true = defined_value == value
366 1.1 christos elif cmp == '!=':
367 1.1 christos is_true = defined_value != value
368 1.1 christos elif cmp == '>=':
369 1.1 christos is_true = defined_value >= value
370 1.1 christos elif cmp == '>':
371 1.1 christos is_true = defined_value > value
372 1.1 christos else:
373 1.1 christos resolved = False
374 1.1 christos
375 1.1 christos if op is not None and not resolved:
376 1.1 christos # Remove the first op in the line + spaces
377 1.1 christos if op == '&&':
378 1.1 christos opre = op
379 1.1 christos else:
380 1.1 christos assert op == '||'
381 1.1 christos opre = r'\|\|'
382 1.1 christos needle = re.compile(fr"(?P<if>\s*#\s*(el)?if\s+).*?(?P<op>{opre}\s*)")
383 1.1 christos match = needle.match(line)
384 1.1 christos assert match is not None
385 1.1 christos newline = line[:match.end('if')] + line[match.end('op'):]
386 1.1 christos
387 1.1 christos self._log(f"\tHardwiring partially resolved {macro}")
388 1.1 christos self._log(f"\t\t- {line[:-1]}")
389 1.1 christos self._log(f"\t\t+ {newline[:-1]}")
390 1.1 christos
391 1.1 christos outlines.append(newline)
392 1.1 christos idx += 1
393 1.1 christos continue
394 1.1 christos
395 1.1 christos # Skip any statements we cannot fully compute
396 1.1 christos if not resolved:
397 1.1 christos outlines.append(line)
398 1.1 christos idx += 1
399 1.1 christos continue
400 1.1 christos
401 1.1 christos prepend = []
402 1.1 christos if macro in self._replaces:
403 1.1 christos assert not ifdef
404 1.1 christos assert op is None
405 1.1 christos value = self._replaces.pop(macro)
406 1.1 christos prepend = [f"#define {macro} {value}\n"]
407 1.1 christos
408 1.1 christos idx, replace = self._handle_if_block(macro, idx, is_true, prepend)
409 1.1 christos outlines += replace
410 1.1 christos changed = True
411 1.1 christos
412 1.1 christos return changed, outlines
413 1.1 christos
414 1.1 christos def preprocess(self, filename):
415 1.1 christos with open(filename, 'r') as f:
416 1.1 christos self._inlines = f.readlines()
417 1.1 christos changed = True
418 1.1 christos iters = 0
419 1.1 christos while changed:
420 1.1 christos iters += 1
421 1.1 christos changed, outlines = self._preprocess_once()
422 1.1 christos self._inlines = outlines
423 1.1 christos
424 1.1 christos with open(filename, 'w') as f:
425 1.1 christos f.write(''.join(self._inlines))
426 1.1 christos
427 1.1 christos
428 1.1 christos class Freestanding(object):
429 1.1 christos def __init__(
430 1.1 christos self, zstd_deps: str, mem: str, source_lib: str, output_lib: str,
431 1.1 christos external_xxhash: bool, xxh64_state: Optional[str],
432 1.1 christos xxh64_prefix: Optional[str], rewritten_includes: [(str, str)],
433 1.1 christos defs: [(str, Optional[str])], replaces: [(str, str)],
434 1.1 christos undefs: [str], excludes: [str], seds: [str], spdx: bool,
435 1.1 christos ):
436 1.1 christos self._zstd_deps = zstd_deps
437 1.1 christos self._mem = mem
438 1.1 christos self._src_lib = source_lib
439 1.1 christos self._dst_lib = output_lib
440 1.1 christos self._external_xxhash = external_xxhash
441 1.1 christos self._xxh64_state = xxh64_state
442 1.1 christos self._xxh64_prefix = xxh64_prefix
443 1.1 christos self._rewritten_includes = rewritten_includes
444 1.1 christos self._defs = defs
445 1.1 christos self._replaces = replaces
446 1.1 christos self._undefs = undefs
447 1.1 christos self._excludes = excludes
448 1.1 christos self._seds = seds
449 1.1 christos self._spdx = spdx
450 1.1 christos
451 1.1 christos def _dst_lib_file_paths(self):
452 1.1 christos """
453 1.1 christos Yields all the file paths in the dst_lib.
454 1.1 christos """
455 1.1 christos for root, dirname, filenames in os.walk(self._dst_lib):
456 1.1 christos for filename in filenames:
457 1.1 christos filepath = os.path.join(root, filename)
458 1.1 christos yield filepath
459 1.1 christos
460 1.1 christos def _log(self, *args, **kwargs):
461 1.1 christos print(*args, **kwargs)
462 1.1 christos
463 1.1 christos def _copy_file(self, lib_path):
464 1.1 christos suffixes = [".c", ".h", ".S"]
465 1.1 christos if not any((lib_path.endswith(suffix) for suffix in suffixes)):
466 1.1 christos return
467 1.1 christos if lib_path in SKIPPED_FILES:
468 1.1 christos self._log(f"\tSkipping file: {lib_path}")
469 1.1 christos return
470 1.1 christos if self._external_xxhash and lib_path in XXHASH_FILES:
471 1.1 christos self._log(f"\tSkipping xxhash file: {lib_path}")
472 1.1 christos return
473 1.1 christos
474 1.1 christos src_path = os.path.join(self._src_lib, lib_path)
475 1.1 christos dst_path = os.path.join(self._dst_lib, lib_path)
476 1.1 christos self._log(f"\tCopying: {src_path} -> {dst_path}")
477 1.1 christos shutil.copyfile(src_path, dst_path)
478 1.1 christos
479 1.1 christos def _copy_source_lib(self):
480 1.1 christos self._log("Copying source library into output library")
481 1.1 christos
482 1.1 christos assert os.path.exists(self._src_lib)
483 1.1 christos os.makedirs(self._dst_lib, exist_ok=True)
484 1.1 christos self._copy_file("zstd.h")
485 1.1 christos self._copy_file("zstd_errors.h")
486 1.1 christos for subdir in INCLUDED_SUBDIRS:
487 1.1 christos src_dir = os.path.join(self._src_lib, subdir)
488 1.1 christos dst_dir = os.path.join(self._dst_lib, subdir)
489 1.1 christos
490 1.1 christos assert os.path.exists(src_dir)
491 1.1 christos os.makedirs(dst_dir, exist_ok=True)
492 1.1 christos
493 1.1 christos for filename in os.listdir(src_dir):
494 1.1 christos lib_path = os.path.join(subdir, filename)
495 1.1 christos self._copy_file(lib_path)
496 1.1 christos
497 1.1 christos def _copy_zstd_deps(self):
498 1.1 christos dst_zstd_deps = os.path.join(self._dst_lib, "common", "zstd_deps.h")
499 1.1 christos self._log(f"Copying zstd_deps: {self._zstd_deps} -> {dst_zstd_deps}")
500 1.1 christos shutil.copyfile(self._zstd_deps, dst_zstd_deps)
501 1.1 christos
502 1.1 christos def _copy_mem(self):
503 1.1 christos dst_mem = os.path.join(self._dst_lib, "common", "mem.h")
504 1.1 christos self._log(f"Copying mem: {self._mem} -> {dst_mem}")
505 1.1 christos shutil.copyfile(self._mem, dst_mem)
506 1.1 christos
507 1.1 christos def _hardwire_preprocessor(self, name: str, value: Optional[str] = None, undef=False):
508 1.1 christos """
509 1.1 christos If value=None then hardwire that it is defined, but not what the value is.
510 1.1 christos If undef=True then value must be None.
511 1.1 christos If value='' then the macro is defined to '' exactly.
512 1.1 christos """
513 1.1 christos assert not (undef and value is not None)
514 1.1 christos for filepath in self._dst_lib_file_paths():
515 1.1 christos file = FileLines(filepath)
516 1.1 christos
517 1.1 christos def _hardwire_defines(self):
518 1.1 christos self._log("Hardwiring macros")
519 1.1 christos partial_preprocessor = PartialPreprocessor(self._defs, self._replaces, self._undefs)
520 1.1 christos for filepath in self._dst_lib_file_paths():
521 1.1 christos partial_preprocessor.preprocess(filepath)
522 1.1 christos
523 1.1 christos def _remove_excludes(self):
524 1.1 christos self._log("Removing excluded sections")
525 1.1 christos for exclude in self._excludes:
526 1.1 christos self._log(f"\tRemoving excluded sections for: {exclude}")
527 1.1 christos begin_re = re.compile(f"BEGIN {exclude}")
528 1.1 christos end_re = re.compile(f"END {exclude}")
529 1.1 christos for filepath in self._dst_lib_file_paths():
530 1.1 christos file = FileLines(filepath)
531 1.1 christos outlines = []
532 1.1 christos skipped = []
533 1.1 christos emit = True
534 1.1 christos for line in file.lines:
535 1.1 christos if emit and begin_re.search(line) is not None:
536 1.1 christos assert end_re.search(line) is None
537 1.1 christos emit = False
538 1.1 christos if emit:
539 1.1 christos outlines.append(line)
540 1.1 christos else:
541 1.1 christos skipped.append(line)
542 1.1 christos if end_re.search(line) is not None:
543 1.1 christos assert begin_re.search(line) is None
544 1.1 christos self._log(f"\t\tRemoving excluded section: {exclude}")
545 1.1 christos for s in skipped:
546 1.1 christos self._log(f"\t\t\t- {s}")
547 1.1 christos emit = True
548 1.1 christos skipped = []
549 1.1 christos if not emit:
550 1.1 christos raise RuntimeError("Excluded section unfinished!")
551 1.1 christos file.lines = outlines
552 1.1 christos file.write()
553 1.1 christos
554 1.1 christos def _rewrite_include(self, original, rewritten):
555 1.1 christos self._log(f"\tRewriting include: {original} -> {rewritten}")
556 1.1 christos regex = re.compile(f"\\s*#\\s*include\\s*(?P<include>{original})")
557 1.1 christos for filepath in self._dst_lib_file_paths():
558 1.1 christos file = FileLines(filepath)
559 1.1 christos for i, line in enumerate(file.lines):
560 1.1 christos match = regex.match(line)
561 1.1 christos if match is None:
562 1.1 christos continue
563 1.1 christos s = match.start('include')
564 1.1 christos e = match.end('include')
565 1.1 christos file.lines[i] = line[:s] + rewritten + line[e:]
566 1.1 christos file.write()
567 1.1 christos
568 1.1 christos def _rewrite_includes(self):
569 1.1 christos self._log("Rewriting includes")
570 1.1 christos for original, rewritten in self._rewritten_includes:
571 1.1 christos self._rewrite_include(original, rewritten)
572 1.1 christos
573 1.1 christos def _replace_xxh64_prefix(self):
574 1.1 christos if self._xxh64_prefix is None:
575 1.1 christos return
576 1.1 christos self._log(f"Replacing XXH64 prefix with {self._xxh64_prefix}")
577 1.1 christos replacements = []
578 1.1 christos if self._xxh64_state is not None:
579 1.1 christos replacements.append(
580 1.1 christos (re.compile(r"([^\w]|^)(?P<orig>XXH64_state_t)([^\w]|$)"), self._xxh64_state)
581 1.1 christos )
582 1.1 christos if self._xxh64_prefix is not None:
583 1.1 christos replacements.append(
584 1.1 christos (re.compile(r"([^\w]|^)(?P<orig>XXH64)[\(_]"), self._xxh64_prefix)
585 1.1 christos )
586 1.1 christos for filepath in self._dst_lib_file_paths():
587 1.1 christos file = FileLines(filepath)
588 1.1 christos for i, line in enumerate(file.lines):
589 1.1 christos modified = False
590 1.1 christos for regex, replacement in replacements:
591 1.1 christos match = regex.search(line)
592 1.1 christos while match is not None:
593 1.1 christos modified = True
594 1.1 christos b = match.start('orig')
595 1.1 christos e = match.end('orig')
596 1.1 christos line = line[:b] + replacement + line[e:]
597 1.1 christos match = regex.search(line)
598 1.1 christos if modified:
599 1.1 christos self._log(f"\t- {file.lines[i][:-1]}")
600 1.1 christos self._log(f"\t+ {line[:-1]}")
601 1.1 christos file.lines[i] = line
602 1.1 christos file.write()
603 1.1 christos
604 1.1 christos def _parse_sed(self, sed):
605 1.1 christos assert sed[0] == 's'
606 1.1 christos delim = sed[1]
607 1.1 christos match = re.fullmatch(f's{delim}(.+){delim}(.*){delim}(.*)', sed)
608 1.1 christos assert match is not None
609 1.1 christos regex = re.compile(match.group(1))
610 1.1 christos format_str = match.group(2)
611 1.1 christos is_global = match.group(3) == 'g'
612 1.1 christos return regex, format_str, is_global
613 1.1 christos
614 1.1 christos def _process_sed(self, sed):
615 1.1 christos self._log(f"Processing sed: {sed}")
616 1.1 christos regex, format_str, is_global = self._parse_sed(sed)
617 1.1 christos
618 1.1 christos for filepath in self._dst_lib_file_paths():
619 1.1 christos file = FileLines(filepath)
620 1.1 christos for i, line in enumerate(file.lines):
621 1.1 christos modified = False
622 1.1 christos while True:
623 1.1 christos match = regex.search(line)
624 1.1 christos if match is None:
625 1.1 christos break
626 1.1 christos replacement = format_str.format(match.groups(''), match.groupdict(''))
627 1.1 christos b = match.start()
628 1.1 christos e = match.end()
629 1.1 christos line = line[:b] + replacement + line[e:]
630 1.1 christos modified = True
631 1.1 christos if not is_global:
632 1.1 christos break
633 1.1 christos if modified:
634 1.1 christos self._log(f"\t- {file.lines[i][:-1]}")
635 1.1 christos self._log(f"\t+ {line[:-1]}")
636 1.1 christos file.lines[i] = line
637 1.1 christos file.write()
638 1.1 christos
639 1.1 christos def _process_seds(self):
640 1.1 christos self._log("Processing seds")
641 1.1 christos for sed in self._seds:
642 1.1 christos self._process_sed(sed)
643 1.1 christos
644 1.1 christos def _process_spdx(self):
645 1.1 christos if not self._spdx:
646 1.1 christos return
647 1.1 christos self._log("Processing spdx")
648 1.1 christos SPDX_C = "// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause\n"
649 1.1 christos SPDX_H_S = "/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */\n"
650 1.1 christos for filepath in self._dst_lib_file_paths():
651 1.1 christos file = FileLines(filepath)
652 1.1 christos if file.lines[0] == SPDX_C or file.lines[0] == SPDX_H_S:
653 1.1 christos continue
654 1.1 christos for line in file.lines:
655 1.1 christos if "SPDX-License-Identifier" in line:
656 1.1 christos raise RuntimeError(f"Unexpected SPDX license identifier: {file.filename} {repr(line)}")
657 1.1 christos if file.filename.endswith(".c"):
658 1.1 christos file.lines.insert(0, SPDX_C)
659 1.1 christos elif file.filename.endswith(".h") or file.filename.endswith(".S"):
660 1.1 christos file.lines.insert(0, SPDX_H_S)
661 1.1 christos else:
662 1.1 christos raise RuntimeError(f"Unexpected file extension: {file.filename}")
663 1.1 christos file.write()
664 1.1 christos
665 1.1 christos
666 1.1 christos
667 1.1 christos def go(self):
668 1.1 christos self._copy_source_lib()
669 1.1 christos self._copy_zstd_deps()
670 1.1 christos self._copy_mem()
671 1.1 christos self._hardwire_defines()
672 1.1 christos self._remove_excludes()
673 1.1 christos self._rewrite_includes()
674 1.1 christos self._replace_xxh64_prefix()
675 1.1 christos self._process_seds()
676 1.1 christos self._process_spdx()
677 1.1 christos
678 1.1 christos
679 1.1 christos def parse_optional_pair(defines: [str]) -> [(str, Optional[str])]:
680 1.1 christos output = []
681 1.1 christos for define in defines:
682 1.1 christos parsed = define.split('=')
683 1.1 christos if len(parsed) == 1:
684 1.1 christos output.append((parsed[0], None))
685 1.1 christos elif len(parsed) == 2:
686 1.1 christos output.append((parsed[0], parsed[1]))
687 1.1 christos else:
688 1.1 christos raise RuntimeError(f"Bad define: {define}")
689 1.1 christos return output
690 1.1 christos
691 1.1 christos
692 1.1 christos def parse_pair(rewritten_includes: [str]) -> [(str, str)]:
693 1.1 christos output = []
694 1.1 christos for rewritten_include in rewritten_includes:
695 1.1 christos parsed = rewritten_include.split('=')
696 1.1 christos if len(parsed) == 2:
697 1.1 christos output.append((parsed[0], parsed[1]))
698 1.1 christos else:
699 1.1 christos raise RuntimeError(f"Bad rewritten include: {rewritten_include}")
700 1.1 christos return output
701 1.1 christos
702 1.1 christos
703 1.1 christos
704 1.1 christos def main(name, args):
705 1.1 christos parser = argparse.ArgumentParser(prog=name)
706 1.1 christos parser.add_argument("--zstd-deps", default="zstd_deps.h", help="Zstd dependencies file")
707 1.1 christos parser.add_argument("--mem", default="mem.h", help="Memory module")
708 1.1 christos parser.add_argument("--source-lib", default="../../lib", help="Location of the zstd library")
709 1.1 christos parser.add_argument("--output-lib", default="./freestanding_lib", help="Where to output the freestanding zstd library")
710 1.1 christos parser.add_argument("--xxhash", default=None, help="Alternate external xxhash include e.g. --xxhash='<xxhash.h>'. If set xxhash is not included.")
711 1.1 christos parser.add_argument("--xxh64-state", default=None, help="Alternate XXH64 state type (excluding _) e.g. --xxh64-state='struct xxh64_state'")
712 1.1 christos parser.add_argument("--xxh64-prefix", default=None, help="Alternate XXH64 function prefix (excluding _) e.g. --xxh64-prefix=xxh64")
713 1.1 christos parser.add_argument("--rewrite-include", default=[], dest="rewritten_includes", action="append", help="Rewrite an include REGEX=NEW (e.g. '<stddef\\.h>=<linux/types.h>')")
714 1.1 christos parser.add_argument("--sed", default=[], dest="seds", action="append", help="Apply a sed replacement. Format: `s/REGEX/FORMAT/[g]`. REGEX is a Python regex. FORMAT is a Python format string formatted by the regex dict.")
715 1.1 christos parser.add_argument("--spdx", action="store_true", help="Add SPDX License Identifiers")
716 1.1 christos parser.add_argument("-D", "--define", default=[], dest="defs", action="append", help="Pre-define this macro (can be passed multiple times)")
717 1.1 christos parser.add_argument("-U", "--undefine", default=[], dest="undefs", action="append", help="Pre-undefine this macro (can be passed multiple times)")
718 1.1 christos parser.add_argument("-R", "--replace", default=[], dest="replaces", action="append", help="Pre-define this macro and replace the first ifndef block with its definition")
719 1.1 christos parser.add_argument("-E", "--exclude", default=[], dest="excludes", action="append", help="Exclude all lines between 'BEGIN <EXCLUDE>' and 'END <EXCLUDE>'")
720 1.1 christos args = parser.parse_args(args)
721 1.1 christos
722 1.1 christos # Always remove threading
723 1.1 christos if "ZSTD_MULTITHREAD" not in args.undefs:
724 1.1 christos args.undefs.append("ZSTD_MULTITHREAD")
725 1.1 christos
726 1.1 christos args.defs = parse_optional_pair(args.defs)
727 1.1 christos for name, _ in args.defs:
728 1.1 christos if name in args.undefs:
729 1.1 christos raise RuntimeError(f"{name} is both defined and undefined!")
730 1.1 christos
731 1.1 christos # Always set tracing to 0
732 1.1 christos if "ZSTD_NO_TRACE" not in (arg[0] for arg in args.defs):
733 1.1 christos args.defs.append(("ZSTD_NO_TRACE", None))
734 1.1 christos args.defs.append(("ZSTD_TRACE", "0"))
735 1.1 christos
736 1.1 christos args.replaces = parse_pair(args.replaces)
737 1.1 christos for name, _ in args.replaces:
738 1.1 christos if name in args.undefs or name in args.defs:
739 1.1 christos raise RuntimeError(f"{name} is both replaced and (un)defined!")
740 1.1 christos
741 1.1 christos args.rewritten_includes = parse_pair(args.rewritten_includes)
742 1.1 christos
743 1.1 christos external_xxhash = False
744 1.1 christos if args.xxhash is not None:
745 1.1 christos external_xxhash = True
746 1.1 christos args.rewritten_includes.append(('"(\\.\\./common/)?xxhash.h"', args.xxhash))
747 1.1 christos
748 1.1 christos if args.xxh64_prefix is not None:
749 1.1 christos if not external_xxhash:
750 1.1 christos raise RuntimeError("--xxh64-prefix may only be used with --xxhash provided")
751 1.1 christos
752 1.1 christos if args.xxh64_state is not None:
753 1.1 christos if not external_xxhash:
754 1.1 christos raise RuntimeError("--xxh64-state may only be used with --xxhash provided")
755 1.1 christos
756 1.1 christos Freestanding(
757 1.1 christos args.zstd_deps,
758 1.1 christos args.mem,
759 1.1 christos args.source_lib,
760 1.1 christos args.output_lib,
761 1.1 christos external_xxhash,
762 1.1 christos args.xxh64_state,
763 1.1 christos args.xxh64_prefix,
764 1.1 christos args.rewritten_includes,
765 1.1 christos args.defs,
766 1.1 christos args.replaces,
767 1.1 christos args.undefs,
768 1.1 christos args.excludes,
769 1.1 christos args.seds,
770 1.1 christos args.spdx,
771 1.1 christos ).go()
772 1.1 christos
773 1.1 christos if __name__ == "__main__":
774 1.1 christos main(sys.argv[0], sys.argv[1:])
775