from dataclasses import dataclass import enum import os from pathlib import Path as ImpurePath, PurePath import re import subprocess import sys from typing import Dict, List, Optional, TextIO, Tuple from .token import * from ..args import Args from ..util import PeekableIterator __all__ = [ 'Makefile', ] @dataclass() class Makefile: _inference_rules: List['InferenceRule'] _macros: Dict[str, Tuple['MacroSource', TokenString]] _targets: Dict[str, 'Target'] args: Args def __init__(self, args: Args): self._inference_rules = [] self._macros = dict() self._targets = dict() self.args = args if args.builtin_rules: self._inference_rules += BUILTIN_INFERENCE_RULES self._macros.update(BUILTIN_MACROS) self._targets.update(BUILTIN_TARGETS) for k, v in os.environ.items(): if k not in ['MAKEFLAGS', 'SHELL']: self._macros[k] = (MacroSource.Environment, TokenString([TextToken(v)])) def read(self, file: TextIO): lines_iter: PeekableIterator[str] = PeekableIterator(iter(file)) for line in lines_iter: # handle escaped newlines (POSIX says these are different in command lines (which we handle later) and # does not define if they are different in include lines (so we treat them as the same) while line.endswith('\\\n'): line = line[:-2] + next(lines_iter, '').lstrip() # POSIX: # > If the word include appears at the beginning of a line and is followed by one or more # > characters... if line.startswith('include '): # > the string formed by the remainder of the line... line = line[len('include '):].lstrip() # > shall be processed as follows to produce a pathname: # > The trailing , any characters immediately preceding a comment, and any comment # > shall be discarded. line = re.sub(r'(\s+#.*)?\n', '', line) # > The resulting string shall be processed for macro expansion. line = self.expand_macros(tokenize(line)) # > Any characters that appear after the first non- shall be used as separators to # > divide the macro-expanded string into fields. fields = line.split() # > If the processing of separators and optional pathname expansion results in either zero or two or # > more non-empty fields, the behavior is unspecified. If it results in one non-empty field, that # > field is taken as the pathname. # (GNU make will include each field separately, so let's do that here) for included_file in fields: # > The contents of the file specified by the pathname shall be read and processed as if they # > appeared in the makefile in place of the include line. self.read(open(included_file, 'r')) # make sure we don't process an ambiguous line as both an include and something else continue # decide if this is a macro or rule line_type = 'unknown' line_tokens = tokenize(line) for t in line_tokens: if isinstance(t, TextToken): if ':' in t.text and ('=' not in t.text or t.text.index(':') < t.text.index('=')): line_type = 'rule' break elif '=' in t.text and (':' not in t.text or t.text.index('=') < t.text.index(':')): line_type = 'macro' break if line_type == 'rule': # > Target entries are specified by a -separated, non-null list of targets, then a , then # > a -separated, possibly empty list of prerequisites. targets, after_colon = line_tokens.split_once(':') targets = self.expand_macros(targets).split() # > Text following a , if any, and all following lines that begin with a , are makefile # > command lines to be executed to update the target. semicolon_split = after_colon.split_once(';') if semicolon_split is None: prerequisites = self.expand_macros(after_colon).split() commands = [] else: prerequisites, commands = semicolon_split prerequisites = self.expand_macros(prerequisites).split() commands = [commands] while lines_iter.peek().startswith('\t'): commands.append(tokenize(next(lines_iter).lstrip('\t'))) commands = [CommandLine(c) for c in commands] # we don't know yet if it's a target rule or an inference rule match = re.fullmatch(r'(?P(\.[^/.]+)?)(?P\.[^/.]+)', targets[0]) if len(targets) == 1 and len(prerequisites) == 0 and match is not None: # it's an inference rule! self._inference_rules.append(InferenceRule(match.group('s1'), match.group('s2'), commands)) else: # it's a target rule! for target in targets: # > A target that has prerequisites, but does not have any commands, can be used to add to the # > prerequisite list for that target. if target in self._targets and len(commands) == 0: self._targets[target].prerequisites += prerequisites else: self._targets[target] = Target(target, prerequisites, commands) elif line_type == 'macro': # > The macro named string1 is defined as having the value of string2, where string2 is defined as all # > characters, if any, after the ... name, value = line_tokens.split_once('=') # > up to a comment character ( '#' ) or an unescaped . comment_split = value.split_once('#') if comment_split is not None: value, _ = comment_split # > Any characters immediately before or after the shall be ignored. name.rstrip() value.lstrip() # > Macros in the string before the in a macro definition shall be evaluated when the # > macro assignment is made. name = self.expand_macros(name) # > Macros defined in the makefile(s) shall override macro definitions that occur before them in the # > makefile(s) and macro definitions from source 4. If the -e option is not specified, macros defined # > in the makefile(s) shall override macro definitions from source 3. Macros defined in the makefile(s) # > shall not override macro definitions from source 1 or source 2. if name in self._macros: source, _ = self._macros[name] inviolate_sources = [MacroSource.CommandLine, MacroSource.MAKEFLAGS] if self.args.environment_overrides: inviolate_sources.append(MacroSource.Environment) if source in inviolate_sources: continue self._macros[name] = (MacroSource.File, value) def expand_macros(self, text: TokenString, current_target: Optional['Target']) -> str: def expand_one(this_token: Token) -> str: if isinstance(this_token, TextToken): return this_token.text elif isinstance(this_token, MacroToken): macro_name = this_token.name internal_macro = len(macro_name) in [1, 2] and macro_name[0] in '@?<*' and \ macro_name[1:] in ['', 'D', 'F'] if internal_macro: if macro_name[0] == '@': # > The $@ shall evaluate to the full target name of the current target, or the archive filename # > part of a library archive target. It shall be evaluated for both target and inference rules. macro_value = [current_target.name] elif macro_name[0] == '?': # > The $? macro shall evaluate to the list of prerequisites that are newer than the current # > target. It shall be evaluated for both target and inference rules. macro_value = [p for p in current_target.prerequisites if self.target(p).newer_than(current_target)] elif macro_name[0] == '<': # > In an inference rule, the $< macro shall evaluate to the filename whose existence allowed # > the inference rule to be chosen for the target. In the .DEFAULT rule, the $< macro shall # > evaluate to the current target name. macro_value = current_target.prerequisites elif macro_name[0] == '*': # > The $* macro shall evaluate to the current target name with its suffix deleted. macro_value = [str(PurePath(current_target.name).with_suffix(''))] else: # this shouldn't happen macro_value = [] if macro_name[1:] == 'D': macro_value = [str(PurePath(x).parent) for x in macro_value] elif macro_name[1:] == 'F': macro_value = [str(PurePath(x).name) for x in macro_value] macro_value = TokenString([TextToken(' '.join(macro_value))]) else: _, macro_value = self._macros[this_token.name] macro_value = self.expand_macros(macro_value, current_target) if this_token.replacement is not None: replaced, replacement = (self.expand_macros(t, current_target) for t in this_token.replacement) macro_value = re.sub(re.escape(replaced) + r'\b', replacement, macro_value) return macro_value return ''.join(expand_one(t) for t in text) def special_target(self, name: str) -> Optional['Target']: return self._targets.get(name, None) def special_target_has_prereq(self, target: str, name: str) -> bool: target = self.special_target(target) if target is None: return False return len(target.prerequisites) == 0 or name in target.prerequisites def target(self, name: str) -> 'Target': # TODO implement .DEFAULT if name not in self._targets: # > When no target rule is found to update a target, the inference rules shall be checked. The suffix of # > the target (.s1) to be built... suffix = PurePath(name).suffix # > is compared to the list of suffixes specified by the .SUFFIXES special targets. If the .s1 suffix is # > found in .SUFFIXES... # (single-suffix rules apply to targets with no suffix so we just throw that in) if self.special_target_has_prereq('.SUFFIXES', suffix) or suffix == '': # > the inference rules shall be searched in the order defined... for rule in self._inference_rules: # > for the first .s2.s1 rule... if rule.s1 == suffix: # > whose prerequisite file ($*.s2) exists. prerequisite_path = PurePath(name).with_suffix(rule.s2) if ImpurePath(prerequisite_path).exists(): self._targets[name] = Target(name, [str(prerequisite_path)], rule.commands) break if name not in self._targets: # we tried inference, it didn't work # is there a default? default = self.special_target('.DEFAULT') if default is not None: self._targets[name] = Target(name, [], default.commands) else: # well, there's no rule available, and no default. does it already exist? if ImpurePath(name).exists(): # it counts as already up to date self._targets[name] = Target(name, [], [], True) return self._targets[name] @dataclass() class InferenceRule: s1: str # empty string means single-suffix rule s2: str commands: List['CommandLine'] @dataclass() class Target: name: str prerequisites: List[str] commands: List['CommandLine'] already_updated: bool = False def _path(self) -> ImpurePath: return ImpurePath(self.name) def modified_time(self) -> Optional[float]: path = self._path() if path.exists(): return path.stat().st_mtime def newer_than(self, other: 'Target') -> Optional[bool]: self_mtime = self.modified_time() other_mtime = other.modified_time() if self_mtime is not None and other_mtime is not None: return self_mtime >= other_mtime elif self_mtime is None and self.already_updated and self.name in other.prerequisites: return True elif other_mtime is None and other.already_updated and other.name in self.prerequisites: return False def is_up_to_date(self, file: Makefile) -> bool: if self.already_updated: return True exists = self._path().exists() newer_than_all_dependencies = all(self.newer_than(file.target(other)) for other in self.prerequisites) return exists and newer_than_all_dependencies def update(self, file: Makefile): for prerequisite in self.prerequisites: file.target(prerequisite).update(file) if not self.is_up_to_date(file): self.execute_commands(file) self.already_updated = True def execute_commands(self, file: Makefile): for command in self.commands: command.execute(file, self) @dataclass() class MacroSource(enum.Enum): File = 0 CommandLine = 1 MAKEFLAGS = 2 Environment = 3 Builtin = 4 @dataclass() class CommandLine: ignore_errors: bool silent: bool always_execute: bool execution_line: TokenString def __init__(self, line: TokenString): self.ignore_errors = False self.silent = False self.always_execute = False # POSIX: # > An execution line is built from the command line by removing any prefix characters. tokens_iter = iter(line) first_token = next(tokens_iter) if isinstance(first_token, TextToken): while first_token.text[0] in ['-', '@', '+']: if first_token.text[0] == '-': self.ignore_errors = True elif first_token.text[0] == '@': self.silent = True elif first_token.text[0] == '+': self.always_execute = True first_token.text = first_token.text[1:] self.execution_line = TokenString(list((first_token, *tokens_iter))) def execute(self, file: Makefile, current_target: 'Target'): # POSIX: # > If the command prefix contains a , or the -i option is present, or the special target .IGNORE # > has either the current target as a prerequisite or has no prerequisites, any error found while executing # > the command shall be ignored. ignore_errors = self.ignore_errors or \ file.args.ignore_errors or \ file.special_target_has_prereq('.IGNORE', current_target.name) # > If the command prefix contains an at-sign and the make utility command line -n option is not specified, or # > the -s option is present, or the special target .SILENT has either the current target as a prerequisite or # > has no prerequisites, the command shall not be written to standard output before it is executed. silent = self.silent and not file.args.dry_run or \ file.args.silent or \ file.special_target_has_prereq('.SILENT', current_target.name) # > If the command prefix contains a , this indicates a makefile command line that shall be executed # > even if -n, -q, or -t is specified. should_execute = self.always_execute or not (file.args.dry_run or file.args.question or file.args.touch) if not should_execute: return execution_line = file.expand_macros(self.execution_line, current_target) # > Except as described under the at-sign prefix... if not silent: # > the execution line shall be written to the standard output. print(execution_line) # > The execution line shall then be executed by a shell as if it were passed as the argument to the system() # > interface, except that if errors are not being ignored then the shell -e option shall also be in effect. # TODO figure out how to pass -e to the shell reliably result = subprocess.call(execution_line, shell=True) # > By default, when make receives a non-zero status from the execution of a command, it shall terminate with # > an error message to standard error. if not ignore_errors and result != 0: print('error!', file=sys.stderr) sys.exit(1) BUILTIN_INFERENCE_RULES = [] BUILTIN_MACROS = {} BUILTIN_TARGETS = {}