aboutsummaryrefslogtreecommitdiff
path: root/yapymake/makefile/__init__.py
blob: 051a9f9dff83a0cb7044dc2dbfbf7655c8b472b3 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
from dataclasses import dataclass
import enum
import os
from pathlib import Path as ImpurePath, PurePath
import re
import subprocess
import sys
from typing import Dict, List, Optional, TextIO, Tuple

from .token import *
from ..args import Args
from ..util import PeekableIterator

__all__ = [
    'Makefile',
]

@dataclass()
class Makefile:
    _inference_rules: List['InferenceRule']
    _macros: Dict[str, Tuple['MacroSource', TokenString]]
    _targets: Dict[str, 'Target']
    args: Args

    def __init__(self, args: Args):
        self._inference_rules = []
        self._macros = dict()
        self._targets = dict()
        self.args = args

        if args.builtin_rules:
            self._inference_rules += BUILTIN_INFERENCE_RULES
            self._macros.update(BUILTIN_MACROS)
            self._targets.update(BUILTIN_TARGETS)

        for k, v in os.environ.items():
            if k not in ['MAKEFLAGS', 'SHELL']:
                self._macros[k] = (MacroSource.Environment, TokenString([TextToken(v)]))

    def read(self, file: TextIO):
        lines_iter: PeekableIterator[str] = PeekableIterator(iter(file))
        for line in lines_iter:
            # handle escaped newlines (POSIX says these are different in command lines (which we handle later) and
            # does not define if they are different in include lines (so we treat them as the same)
            while line.endswith('\\\n'):
                line = line[:-2] + next(lines_iter, '').lstrip()

            # POSIX:
            # > If the word include appears at the beginning of a line and is followed by one or more <blank>
            # > characters...
            if line.startswith('include '):
                # > the string formed by the remainder of the line...
                line = line[len('include '):].lstrip()
                # > shall be processed as follows to produce a pathname:

                # > The trailing <newline>, any <blank> characters immediately preceding a comment, and any comment
                # > shall be discarded.
                line = re.sub(r'(\s+#.*)?\n', '', line)

                # > The resulting string shall be processed for macro expansion.
                line = self.expand_macros(tokenize(line))

                # > Any <blank> characters that appear after the first non- <blank> shall be used as separators to
                # > divide the macro-expanded string into fields.
                fields = line.split()

                # > If the processing of separators and optional pathname expansion results in either zero or two or
                # > more non-empty fields, the behavior is unspecified. If it results in one non-empty field, that
                # > field is taken as the pathname.
                # (GNU make will include each field separately, so let's do that here)
                for included_file in fields:
                    # > The contents of the file specified by the pathname shall be read and processed as if they
                    # > appeared in the makefile in place of the include line.
                    self.read(open(included_file, 'r'))

                # make sure we don't process an ambiguous line as both an include and something else
                continue

            # decide if this is a macro or rule
            line_type = 'unknown'
            line_tokens = tokenize(line)
            for t in line_tokens:
                if isinstance(t, TextToken):
                    if ':' in t.text and ('=' not in t.text or t.text.index(':') < t.text.index('=')):
                        line_type = 'rule'
                        break
                    elif '=' in t.text and (':' not in t.text or t.text.index('=') < t.text.index(':')):
                        line_type = 'macro'
                        break

            if line_type == 'rule':
                # > Target entries are specified by a <blank>-separated, non-null list of targets, then a <colon>, then
                # > a <blank>-separated, possibly empty list of prerequisites.
                targets, after_colon = line_tokens.split_once(':')
                targets = self.expand_macros(targets).split()
                # > Text following a <semicolon>, if any, and all following lines that begin with a <tab>, are makefile
                # > command lines to be executed to update the target.
                semicolon_split = after_colon.split_once(';')
                if semicolon_split is None:
                    prerequisites = self.expand_macros(after_colon).split()
                    commands = []
                else:
                    prerequisites, commands = semicolon_split
                    prerequisites = self.expand_macros(prerequisites).split()
                    commands = [commands]
                while lines_iter.peek().startswith('\t'):
                    commands.append(tokenize(next(lines_iter).lstrip('\t')))
                commands = [CommandLine(c) for c in commands]

                # we don't know yet if it's a target rule or an inference rule
                match = re.fullmatch(r'(?P<s1>(\.[^/.]+)?)(?P<s2>\.[^/.]+)', targets[0])
                if len(targets) == 1 and len(prerequisites) == 0 and match is not None:
                    # it's an inference rule!
                    self._inference_rules.append(InferenceRule(match.group('s1'), match.group('s2'), commands))
                else:
                    # it's a target rule!
                    for target in targets:
                        # > A target that has prerequisites, but does not have any commands, can be used to add to the
                        # > prerequisite list for that target.
                        if target in self._targets and len(commands) == 0:
                            self._targets[target].prerequisites += prerequisites
                        else:
                            self._targets[target] = Target(target, prerequisites, commands)
            elif line_type == 'macro':
                # > The macro named string1 is defined as having the value of string2, where string2 is defined as all
                # > characters, if any, after the <equals-sign>...
                name, value = line_tokens.split_once('=')
                # > up to a comment character ( '#' ) or an unescaped <newline>.
                comment_split = value.split_once('#')
                if comment_split is not None:
                    value, _ = comment_split
                # > Any <blank> characters immediately before or after the <equals-sign> shall be ignored.
                name.rstrip()
                value.lstrip()
                # > Macros in the string before the <equals-sign> in a macro definition shall be evaluated when the
                # > macro assignment is made.
                name = self.expand_macros(name)
                # > Macros defined in the makefile(s) shall override macro definitions that occur before them in the
                # > makefile(s) and macro definitions from source 4. If the -e option is not specified, macros defined
                # > in the makefile(s) shall override macro definitions from source 3. Macros defined in the makefile(s)
                # > shall not override macro definitions from source 1 or source 2.
                if name in self._macros:
                    source, _ = self._macros[name]
                    inviolate_sources = [MacroSource.CommandLine, MacroSource.MAKEFLAGS]
                    if self.args.environment_overrides:
                        inviolate_sources.append(MacroSource.Environment)
                    if source in inviolate_sources:
                        continue
                self._macros[name] = (MacroSource.File, value)

    def expand_macros(self, text: TokenString, current_target: Optional['Target']) -> str:
        def expand_one(this_token: Token) -> str:
            if isinstance(this_token, TextToken):
                return this_token.text
            elif isinstance(this_token, MacroToken):
                macro_name = this_token.name
                internal_macro = len(macro_name) in [1, 2] and macro_name[0] in '@?<*' and \
                                 macro_name[1:] in ['', 'D', 'F']
                if internal_macro:
                    if macro_name[0] == '@':
                        # > The $@ shall evaluate to the full target name of the current target, or the archive filename
                        # > part of a library archive target. It shall be evaluated for both target and inference rules.
                        macro_value = [current_target.name]
                    elif macro_name[0] == '?':
                        # > The $? macro shall evaluate to the list of prerequisites that are newer than the current
                        # > target. It shall be evaluated for both target and inference rules.
                        macro_value = [p for p in current_target.prerequisites if self.target(p).newer_than(current_target)]
                    elif macro_name[0] == '<':
                        # > In an inference rule, the $< macro shall evaluate to the filename whose existence allowed
                        # > the inference rule to be chosen for the target. In the .DEFAULT rule, the $< macro shall
                        # > evaluate to the current target name.
                        macro_value = current_target.prerequisites
                    elif macro_name[0] == '*':
                        # > The $* macro shall evaluate to the current target name with its suffix deleted.
                        macro_value = [str(PurePath(current_target.name).with_suffix(''))]
                    else:
                        # this shouldn't happen
                        macro_value = []

                    if macro_name[1:] == 'D':
                        macro_value = [str(PurePath(x).parent) for x in macro_value]
                    elif macro_name[1:] == 'F':
                        macro_value = [str(PurePath(x).name) for x in macro_value]

                    macro_value = TokenString([TextToken(' '.join(macro_value))])
                else:
                    _, macro_value = self._macros[this_token.name]
                macro_value = self.expand_macros(macro_value, current_target)
                if this_token.replacement is not None:
                    replaced, replacement = (self.expand_macros(t, current_target) for t in this_token.replacement)
                    macro_value = re.sub(re.escape(replaced) + r'\b', replacement, macro_value)
                return macro_value

        return ''.join(expand_one(t) for t in text)

    def special_target(self, name: str) -> Optional['Target']:
        return self._targets.get(name, None)

    def special_target_has_prereq(self, target: str, name: str) -> bool:
        target = self.special_target(target)
        if target is None:
            return False
        return len(target.prerequisites) == 0 or name in target.prerequisites

    def target(self, name: str) -> 'Target':
        # TODO implement .DEFAULT
        if name not in self._targets:
            # > When no target rule is found to update a target, the inference rules shall be checked. The suffix of
            # > the target (.s1) to be built...
            suffix = PurePath(name).suffix
            # > is compared to the list of suffixes specified by the .SUFFIXES special targets. If the .s1 suffix is
            # > found in .SUFFIXES...
            # (single-suffix rules apply to targets with no suffix so we just throw that in)
            if self.special_target_has_prereq('.SUFFIXES', suffix) or suffix == '':
                # > the inference rules shall be searched in the order defined...
                for rule in self._inference_rules:
                    # > for the first .s2.s1 rule...
                    if rule.s1 == suffix:
                        # > whose prerequisite file ($*.s2) exists.
                        prerequisite_path = PurePath(name).with_suffix(rule.s2)
                        if ImpurePath(prerequisite_path).exists():
                            self._targets[name] = Target(name, [str(prerequisite_path)], rule.commands)
                            break
        if name not in self._targets:
            # we tried inference, it didn't work
            # is there a default?
            default = self.special_target('.DEFAULT')
            if default is not None:
                self._targets[name] = Target(name, [], default.commands)
            else:
                # well, there's no rule available, and no default. does it already exist?
                if ImpurePath(name).exists():
                    # it counts as already up to date
                    self._targets[name] = Target(name, [], [], True)
        return self._targets[name]

@dataclass()
class InferenceRule:
    s1: str # empty string means single-suffix rule
    s2: str
    commands: List['CommandLine']

@dataclass()
class Target:
    name: str
    prerequisites: List[str]
    commands: List['CommandLine']
    already_updated: bool = False

    def _path(self) -> ImpurePath:
        return ImpurePath(self.name)

    def modified_time(self) -> Optional[float]:
        path = self._path()
        if path.exists():
            return path.stat().st_mtime

    def newer_than(self, other: 'Target') -> Optional[bool]:
        self_mtime = self.modified_time()
        other_mtime = other.modified_time()
        if self_mtime is not None and other_mtime is not None:
            return self_mtime >= other_mtime
        elif self_mtime is None and self.already_updated and self.name in other.prerequisites:
            return True
        elif other_mtime is None and other.already_updated and other.name in self.prerequisites:
            return False

    def is_up_to_date(self, file: Makefile) -> bool:
        if self.already_updated:
            return True
        exists = self._path().exists()
        newer_than_all_dependencies = all(self.newer_than(file.target(other)) for other in self.prerequisites)
        return exists and newer_than_all_dependencies

    def update(self, file: Makefile):
        for prerequisite in self.prerequisites:
            file.target(prerequisite).update(file)
        if not self.is_up_to_date(file):
            self.execute_commands(file)
        self.already_updated = True

    def execute_commands(self, file: Makefile):
        for command in self.commands:
            command.execute(file, self)

@dataclass()
class MacroSource(enum.Enum):
    File = 0
    CommandLine = 1
    MAKEFLAGS = 2
    Environment = 3
    Builtin = 4

@dataclass()
class CommandLine:
    ignore_errors: bool
    silent: bool
    always_execute: bool
    execution_line: TokenString

    def __init__(self, line: TokenString):
        self.ignore_errors = False
        self.silent = False
        self.always_execute = False

        # POSIX:
        # > An execution line is built from the command line by removing any prefix characters.
        tokens_iter = iter(line)
        first_token = next(tokens_iter)
        if isinstance(first_token, TextToken):
            while first_token.text[0] in ['-', '@', '+']:
                if first_token.text[0] == '-':
                    self.ignore_errors = True
                elif first_token.text[0] == '@':
                    self.silent = True
                elif first_token.text[0] == '+':
                    self.always_execute = True
                first_token.text = first_token.text[1:]
        self.execution_line = TokenString(list((first_token, *tokens_iter)))

    def execute(self, file: Makefile, current_target: 'Target'):
        # POSIX:
        # > If the command prefix contains a <hyphen-minus>, or the -i option is present, or the special target .IGNORE
        # > has either the current target as a prerequisite or has no prerequisites, any error found while executing
        # > the command shall be ignored.
        ignore_errors = self.ignore_errors or \
                        file.args.ignore_errors or \
                        file.special_target_has_prereq('.IGNORE', current_target.name)

        # > If the command prefix contains an at-sign and the make utility command line -n option is not specified, or
        # > the -s option is present, or the special target .SILENT has either the current target as a prerequisite or
        # > has no prerequisites, the command shall not be written to standard output before it is executed.
        silent = self.silent and not file.args.dry_run or \
                 file.args.silent or \
                 file.special_target_has_prereq('.SILENT', current_target.name)

        # > If the command prefix contains a <plus-sign>, this indicates a makefile command line that shall be executed
        # > even if -n, -q, or -t is specified.
        should_execute = self.always_execute or not (file.args.dry_run or file.args.question or file.args.touch)
        if not should_execute:
            return

        execution_line = file.expand_macros(self.execution_line, current_target)

        # > Except as described under the at-sign prefix...
        if not silent:
            # > the execution line shall be written to the standard output.
            print(execution_line)

        # > The execution line shall then be executed by a shell as if it were passed as the argument to the system()
        # > interface, except that if errors are not being ignored then the shell -e option shall also be in effect.
        # TODO figure out how to pass -e to the shell reliably
        result = subprocess.call(execution_line, shell=True)

        # > By default, when make receives a non-zero status from the execution of a command, it shall terminate with
        # > an error message to standard error.
        if not ignore_errors and result != 0:
            print('error!', file=sys.stderr)
            sys.exit(1)

BUILTIN_INFERENCE_RULES = []
BUILTIN_MACROS = {}
BUILTIN_TARGETS = {}