Merge the deduplication and dead-code removal pass
Consolidates duplicated logic behind shared implementations — paginated grid tabs, focus chrome, cached remote stores, sheet selection columns, the server artifact store and a test fixture layer — and removes code that had become unreachable. Net reduction of about 5,500 lines with no behaviour change. Where a fix had landed separately in code that moved into a shared helper, the fix was re-applied inside the helper rather than left behind in the copy that went away.
This commit is contained in:
edde746
@@ -5,8 +5,15 @@ from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
from workflow_yaml import iter_uses_references, job_block
|
||||
|
||||
DEFAULT_WORKFLOW = Path(__file__).resolve().parents[1] / ".github/workflows/build.yml"
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_WORKFLOW = ROOT / ".github/workflows/build.yml"
|
||||
# The shared bootstrap both windows-arm jobs call, and the pins it must keep.
|
||||
SETUP_FLUTTER_GIT = ROOT / ".github/actions/setup-flutter-git/action.yml"
|
||||
FLUTTER_VERSION = "3.44.0"
|
||||
FLUTTER_COMMIT = "559ffa3f75e7402d65a8def9c28389a9b2e6fe42"
|
||||
if len(sys.argv) > 2:
|
||||
raise SystemExit(f"Usage: {Path(sys.argv[0]).name} [workflow-path]")
|
||||
WORKFLOW = Path(sys.argv[1]).resolve() if len(sys.argv) == 2 else DEFAULT_WORKFLOW
|
||||
@@ -20,11 +27,9 @@ def require(condition: bool, message: str) -> None:
|
||||
|
||||
|
||||
def job(name: str) -> str:
|
||||
match = re.search(
|
||||
rf"(?ms)^ {re.escape(name)}:\n(.*?)(?=^ [a-zA-Z0-9_-]+:\n|\Z)", text
|
||||
)
|
||||
require(match is not None, f"missing {name} job")
|
||||
return match.group(0) if match else ""
|
||||
block = job_block(text, name)
|
||||
require(bool(block), f"missing {name} job")
|
||||
return block
|
||||
|
||||
|
||||
def named_step(block: str, name: str) -> str:
|
||||
@@ -132,13 +137,7 @@ require(
|
||||
for expected in (
|
||||
"if: matrix.flutter_setup == 'action'",
|
||||
"if: matrix.flutter_setup == 'git'",
|
||||
'$expectedCommit = "559ffa3f75e7402d65a8def9c28389a9b2e6fe42"',
|
||||
"git -C $root fetch --depth 1 origin refs/tags/3.44.0:refs/tags/3.44.0",
|
||||
"git -C $root checkout --detach refs/tags/3.44.0",
|
||||
"$actualCommit = git -C $root rev-parse HEAD",
|
||||
"$actualCommit -ne $expectedCommit",
|
||||
r'$versionOutput = & "$root\bin\flutter.bat" --version --machine',
|
||||
'$version.frameworkVersion -ne "3.44.0"',
|
||||
"uses: ./.github/actions/setup-flutter-git",
|
||||
"flutter pub get --enforce-lockfile --no-example",
|
||||
"--dart-define=SENTRY_DIST=github-windows-${{ matrix.arch }}",
|
||||
"--split-debug-info=debug-info/windows-${{ matrix.arch }}",
|
||||
@@ -160,6 +159,28 @@ require(
|
||||
)
|
||||
require_explicit_shells("build-windows", windows, "pwsh")
|
||||
|
||||
setup_flutter_git = (
|
||||
SETUP_FLUTTER_GIT.read_text(encoding="utf-8") if SETUP_FLUTTER_GIT.is_file() else ""
|
||||
)
|
||||
require(bool(setup_flutter_git), "missing .github/actions/setup-flutter-git/action.yml")
|
||||
for expected in (
|
||||
f'$version = "{FLUTTER_VERSION}"',
|
||||
f'$expectedCommit = "{FLUTTER_COMMIT}"',
|
||||
# Fetch the release tag rather than the bare commit: the commit is only
|
||||
# reachable through the tag, and the tag is what makes the SDK report its
|
||||
# own version. Both halves are then verified, so a moved tag fails the job.
|
||||
'git -C $root fetch --depth 1 origin "refs/tags/${version}:refs/tags/${version}"',
|
||||
'git -C $root checkout --detach "refs/tags/$version"',
|
||||
"$actualCommit = git -C $root rev-parse HEAD",
|
||||
"$actualCommit -ne $expectedCommit",
|
||||
r'$versionOutput = & "$root\bin\flutter.bat" --version --machine',
|
||||
"$reportedVersion -ne $version",
|
||||
):
|
||||
require(
|
||||
expected in setup_flutter_git,
|
||||
f"shared Flutter bootstrap must keep its verified pin: {expected}",
|
||||
)
|
||||
|
||||
linux = job("build-linux")
|
||||
require("runs-on: ${{ matrix.runner }}" in linux, "Linux must use its matrix runner")
|
||||
require("fail-fast: false" in linux, "Linux matrix must not cancel its other architecture")
|
||||
@@ -187,7 +208,7 @@ require(
|
||||
)
|
||||
for expected in (
|
||||
"channel: ${{ matrix.flutter_channel }}",
|
||||
'flutter-version: "3.44.0"',
|
||||
"flutter-version: ${{ env.FLUTTER_VERSION }}",
|
||||
"flutter pub get --enforce-lockfile --no-example",
|
||||
"lib/${{ matrix.pkg_config_arch }}/pkgconfig",
|
||||
"--dart-define=SENTRY_DIST=github-linux-${{ matrix.arch }}",
|
||||
@@ -292,6 +313,10 @@ for protected_job in (
|
||||
f"{protected_job} must depend on trusted-ref validation",
|
||||
)
|
||||
|
||||
require(
|
||||
text.count(FLUTTER_VERSION) == 1 and f'FLUTTER_VERSION: "{FLUTTER_VERSION}"' in text,
|
||||
"the Flutter SDK version must be written once, as the workflow FLUTTER_VERSION env",
|
||||
)
|
||||
require(
|
||||
"TRUSTED_BUILD_CACHE_VERSION: trusted-build-v1" in text,
|
||||
"build caches must use a dedicated trusted namespace",
|
||||
@@ -309,17 +334,18 @@ require(
|
||||
"every Flutter SDK cache must define its trusted cache key",
|
||||
)
|
||||
|
||||
action_refs = re.findall(r"(?m)^\s*(?:-\s+)?uses:\s+([^\s@]+)@([^\s#]+)", text)
|
||||
require(bool(action_refs), "build workflow must use pinned actions")
|
||||
for action, ref in action_refs:
|
||||
require(
|
||||
re.fullmatch(r"[0-9a-f]{40}", ref) is not None,
|
||||
f"action {action} must be pinned to a full commit SHA",
|
||||
)
|
||||
|
||||
checkout_count = sum(action == "actions/checkout" for action, _ in action_refs)
|
||||
# check_workflow_action_pins.py owns the SHA-pin rule for every workflow, this
|
||||
# one included; build.yml only adds the credential invariant on top, because it
|
||||
# is workflow_dispatch-only and so escapes the pull-request rule in
|
||||
# check_workflow_security.py.
|
||||
remote_actions = [
|
||||
reference.rpartition("@")[0]
|
||||
for _, reference in iter_uses_references(text)
|
||||
if not reference.startswith("./")
|
||||
]
|
||||
require(bool(remote_actions), "build workflow must use pinned actions")
|
||||
require(
|
||||
text.count("persist-credentials: false") == checkout_count,
|
||||
text.count("persist-credentials: false") == remote_actions.count("actions/checkout"),
|
||||
"every build checkout must discard GitHub credentials",
|
||||
)
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
from workflow_yaml import job_block
|
||||
|
||||
|
||||
WORKFLOW = Path(__file__).resolve().parents[1] / ".github/workflows/update-packages.yml"
|
||||
text = WORKFLOW.read_text(encoding="utf-8")
|
||||
@@ -17,11 +19,9 @@ def require(condition: bool, message: str) -> None:
|
||||
|
||||
|
||||
def job(name: str) -> str:
|
||||
match = re.search(
|
||||
rf"(?ms)^ {re.escape(name)}:\n(.*?)(?=^ [a-zA-Z0-9_-]+:\n|\Z)", text
|
||||
)
|
||||
require(match is not None, f"missing {name} job")
|
||||
return match.group(0) if match else ""
|
||||
block = job_block(text, name)
|
||||
require(bool(block), f"missing {name} job")
|
||||
return block
|
||||
|
||||
|
||||
require(
|
||||
|
||||
@@ -7,336 +7,21 @@ import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import workflow_yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
WORKFLOWS = ROOT / ".github" / "workflows"
|
||||
MAPPING_RE = re.compile(
|
||||
r"""^\s*(?:-\s*)?(?P<key>uses|'(?:''|[^'])*'|"(?:\\.|[^"\\])*")\s*:\s*(?P<value>.*?)\s*$"""
|
||||
)
|
||||
EXPLICIT_KEY_RE = re.compile(
|
||||
r"""^\s*(?:-\s*)?\?\s*(?P<key>uses|'(?:''|[^'])*'|"(?:\\.|[^"\\])*")\s*$"""
|
||||
)
|
||||
EXPLICIT_VALUE_RE = re.compile(r"^\s*:\s*(?P<value>.*?)\s*$")
|
||||
ACTIONS = ROOT / ".github" / "actions"
|
||||
REMOTE_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_./-]+)?@[0-9a-fA-F]{40}$")
|
||||
BLOCK_SCALAR_RE = re.compile(r":\s*[|>](?:[1-9][+-]?|[+-][1-9]?)?\s*(?:#.*)?$")
|
||||
BLOCK_SCALAR_VALUE_RE = re.compile(r"^[|>](?:[1-9][+-]?|[+-][1-9]?)?$")
|
||||
YAML_DOUBLE_ESCAPES = {
|
||||
"0": "\0",
|
||||
"a": "\a",
|
||||
"b": "\b",
|
||||
"t": "\t",
|
||||
"\t": "\t",
|
||||
"n": "\n",
|
||||
"v": "\v",
|
||||
"f": "\f",
|
||||
"r": "\r",
|
||||
"e": "\x1b",
|
||||
" ": " ",
|
||||
'"': '"',
|
||||
"/": "/",
|
||||
"\\": "\\",
|
||||
"N": "\u0085",
|
||||
"_": "\u00a0",
|
||||
"L": "\u2028",
|
||||
"P": "\u2029",
|
||||
}
|
||||
|
||||
|
||||
def iter_workflow_files(directory: Path = WORKFLOWS):
|
||||
yield from sorted((*directory.glob("*.yml"), *directory.glob("*.yaml")))
|
||||
|
||||
|
||||
def _strip_yaml_comment(value: str) -> str:
|
||||
quote = None
|
||||
escaped = False
|
||||
for index, char in enumerate(value):
|
||||
if escaped:
|
||||
escaped = False
|
||||
continue
|
||||
if char == "\\" and quote == '"':
|
||||
escaped = True
|
||||
continue
|
||||
if char in ("'", '"'):
|
||||
if quote is None:
|
||||
quote = char
|
||||
elif quote == char:
|
||||
quote = None
|
||||
continue
|
||||
if char == "#" and quote is None and (index == 0 or value[index - 1].isspace()):
|
||||
return value[:index].rstrip()
|
||||
return value.rstrip()
|
||||
|
||||
|
||||
def _decode_quoted_yaml_string(value: str) -> str | None:
|
||||
if len(value) < 2 or value[0] != value[-1] or value[0] not in ("'", '"'):
|
||||
return None
|
||||
if value[0] == "'":
|
||||
return value[1:-1].replace("''", "'")
|
||||
|
||||
decoded = []
|
||||
index = 1
|
||||
end = len(value) - 1
|
||||
while index < end:
|
||||
char = value[index]
|
||||
if char != "\\":
|
||||
decoded.append(char)
|
||||
index += 1
|
||||
continue
|
||||
index += 1
|
||||
if index >= end:
|
||||
return None
|
||||
escape = value[index]
|
||||
if escape in YAML_DOUBLE_ESCAPES:
|
||||
decoded.append(YAML_DOUBLE_ESCAPES[escape])
|
||||
index += 1
|
||||
continue
|
||||
width = {"x": 2, "u": 4, "U": 8}.get(escape)
|
||||
if width is None or index + width >= end:
|
||||
return None
|
||||
digits = value[index + 1 : index + 1 + width]
|
||||
if not re.fullmatch(rf"[0-9a-fA-F]{{{width}}}", digits):
|
||||
return None
|
||||
try:
|
||||
decoded.append(chr(int(digits, 16)))
|
||||
except ValueError:
|
||||
return None
|
||||
index += width + 1
|
||||
return "".join(decoded)
|
||||
|
||||
|
||||
def _unquote(value: str) -> str:
|
||||
decoded = _decode_quoted_yaml_string(value)
|
||||
return value if decoded is None else decoded
|
||||
|
||||
|
||||
def _flow_value(line: str, start: int, mapping_depth: int) -> str:
|
||||
index = start
|
||||
quote = None
|
||||
escaped = False
|
||||
depth = mapping_depth
|
||||
while index < len(line):
|
||||
char = line[index]
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\" and quote == '"':
|
||||
escaped = True
|
||||
elif quote is not None:
|
||||
if char == quote:
|
||||
quote = None
|
||||
elif char in ("'", '"'):
|
||||
quote = char
|
||||
elif char in ("{", "["):
|
||||
depth += 1
|
||||
elif char in ("}", "]"):
|
||||
if depth == mapping_depth:
|
||||
break
|
||||
depth -= 1
|
||||
elif char == "," and depth == mapping_depth:
|
||||
break
|
||||
index += 1
|
||||
return _unquote(line[start:index].strip())
|
||||
|
||||
|
||||
def _has_unsupported_block_mapping_key(line: str) -> bool:
|
||||
candidate = line.lstrip()
|
||||
if candidate.startswith("-") and not candidate.startswith("---"):
|
||||
candidate = candidate[1:].lstrip()
|
||||
if not candidate:
|
||||
return False
|
||||
if candidate[0] in "!&*":
|
||||
return True
|
||||
if candidate[0] not in ("'", '"'):
|
||||
return False
|
||||
|
||||
quote = candidate[0]
|
||||
escaped = False
|
||||
index = 1
|
||||
while index < len(candidate):
|
||||
char = candidate[index]
|
||||
if quote == "'" and char == "'" and index + 1 < len(candidate) and candidate[index + 1] == "'":
|
||||
index += 2
|
||||
continue
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif quote == '"' and char == "\\":
|
||||
escaped = True
|
||||
elif char == quote:
|
||||
return False
|
||||
index += 1
|
||||
return True
|
||||
|
||||
|
||||
def _flow_uses_references(line: str, initial_depth: int) -> tuple[list[str], int]:
|
||||
references = []
|
||||
depth = initial_depth
|
||||
index = 0
|
||||
while index < len(line):
|
||||
char = line[index]
|
||||
if char in ("'", '"'):
|
||||
quote = char
|
||||
escaped = False
|
||||
end = index + 1
|
||||
while end < len(line):
|
||||
quoted_char = line[end]
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif quoted_char == "\\" and quote == '"':
|
||||
escaped = True
|
||||
elif quoted_char == quote:
|
||||
break
|
||||
end += 1
|
||||
if end >= len(line):
|
||||
if depth > 0:
|
||||
references.append("<unsupported multiline flow scalar>")
|
||||
return references, depth
|
||||
key = _decode_quoted_yaml_string(line[index : end + 1])
|
||||
after_key = end + 1
|
||||
while after_key < len(line) and line[after_key].isspace():
|
||||
after_key += 1
|
||||
if depth > 0 and after_key < len(line) and line[after_key] == ":":
|
||||
if key == "uses":
|
||||
references.append(_flow_value(line, after_key + 1, depth))
|
||||
elif key is None:
|
||||
references.append("<unsupported quoted flow mapping key>")
|
||||
index = end + 1
|
||||
continue
|
||||
if line.startswith("${{", index):
|
||||
expression_end = line.find("}}", index + 3)
|
||||
if expression_end < 0:
|
||||
references.append("<unterminated GitHub expression>")
|
||||
return references, depth
|
||||
index = expression_end + 2
|
||||
continue
|
||||
if char in ("{", "["):
|
||||
depth += 1
|
||||
index += 1
|
||||
continue
|
||||
if char in ("}", "]"):
|
||||
depth = max(0, depth - 1)
|
||||
index += 1
|
||||
continue
|
||||
if depth > 0 and char == "?":
|
||||
references.append("<unsupported explicit flow mapping>")
|
||||
index += 1
|
||||
continue
|
||||
if depth > 0 and char in "!&*":
|
||||
references.append("<unsupported tagged, anchored, or aliased flow mapping>")
|
||||
index += 1
|
||||
continue
|
||||
if depth > 0 and (char.isalpha() or char == "_"):
|
||||
end = index + 1
|
||||
while end < len(line) and (line[end].isalnum() or line[end] in "_-"):
|
||||
end += 1
|
||||
after_key = end
|
||||
while after_key < len(line) and line[after_key].isspace():
|
||||
after_key += 1
|
||||
if line[index:end] == "uses" and after_key < len(line) and line[after_key] == ":":
|
||||
references.append(_flow_value(line, after_key + 1, depth))
|
||||
index = end
|
||||
continue
|
||||
index += 1
|
||||
return references, depth
|
||||
def iter_action_files(directory: Path = ACTIONS):
|
||||
"""Local composite actions run in the same trust boundary as the workflows."""
|
||||
yield from sorted((*directory.glob("*/action.yml"), *directory.glob("*/action.yaml")))
|
||||
|
||||
|
||||
def iter_uses_references(path: Path):
|
||||
block_parent_indent = None
|
||||
block_content_indent = None
|
||||
block_uses_line = None
|
||||
block_uses_content: list[str] = []
|
||||
explicit_uses_line = None
|
||||
flow_start_line = None
|
||||
flow_depth = 0
|
||||
for line_number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
stripped = raw_line.lstrip()
|
||||
indent = len(raw_line) - len(stripped)
|
||||
if block_parent_indent is not None:
|
||||
if not stripped:
|
||||
if block_uses_line is not None:
|
||||
block_uses_content.append("")
|
||||
continue
|
||||
if indent <= block_parent_indent:
|
||||
if block_uses_line is not None:
|
||||
yield block_uses_line, "\n".join(block_uses_content).strip()
|
||||
block_parent_indent = None
|
||||
block_content_indent = None
|
||||
block_uses_line = None
|
||||
block_uses_content = []
|
||||
elif block_content_indent is None:
|
||||
block_content_indent = indent
|
||||
if block_uses_line is not None:
|
||||
block_uses_content.append(raw_line[block_content_indent:])
|
||||
continue
|
||||
elif indent >= block_content_indent:
|
||||
if block_uses_line is not None:
|
||||
block_uses_content.append(raw_line[block_content_indent:])
|
||||
continue
|
||||
else:
|
||||
if block_uses_line is not None:
|
||||
yield block_uses_line, "\n".join(block_uses_content).strip()
|
||||
block_parent_indent = None
|
||||
block_content_indent = None
|
||||
block_uses_line = None
|
||||
block_uses_content = []
|
||||
if stripped.startswith("#") or not stripped:
|
||||
continue
|
||||
active_line = _strip_yaml_comment(raw_line)
|
||||
if explicit_uses_line is not None:
|
||||
explicit_value = EXPLICIT_VALUE_RE.match(active_line)
|
||||
if explicit_value is None:
|
||||
yield explicit_uses_line, "<missing explicit mapping value>"
|
||||
else:
|
||||
value = explicit_value.group("value").strip()
|
||||
if BLOCK_SCALAR_VALUE_RE.fullmatch(value):
|
||||
block_parent_indent = indent
|
||||
block_content_indent = None
|
||||
block_uses_line = explicit_uses_line
|
||||
block_uses_content = []
|
||||
else:
|
||||
yield explicit_uses_line, _unquote(value)
|
||||
explicit_uses_line = None
|
||||
continue
|
||||
explicit_uses_line = None
|
||||
explicit_key = EXPLICIT_KEY_RE.match(active_line)
|
||||
if explicit_key:
|
||||
if _unquote(explicit_key.group("key")) == "uses":
|
||||
explicit_uses_line = line_number
|
||||
continue
|
||||
if re.match(r"^\s*(?:-\s*)?\?", active_line):
|
||||
yield line_number, "<unsupported explicit mapping key>"
|
||||
continue
|
||||
if _has_unsupported_block_mapping_key(active_line):
|
||||
yield line_number, "<unsupported multiline, tagged, anchored, or aliased mapping key>"
|
||||
continue
|
||||
match = MAPPING_RE.match(active_line) if flow_depth == 0 else None
|
||||
if match:
|
||||
key = _unquote(match.group("key"))
|
||||
value = match.group("value").strip()
|
||||
if BLOCK_SCALAR_VALUE_RE.fullmatch(value):
|
||||
block_parent_indent = indent
|
||||
block_content_indent = None
|
||||
if key == "uses":
|
||||
block_uses_line = line_number
|
||||
block_uses_content = []
|
||||
continue
|
||||
if key == "uses":
|
||||
yield line_number, _unquote(value)
|
||||
if BLOCK_SCALAR_RE.search(raw_line):
|
||||
block_parent_indent = indent
|
||||
block_content_indent = None
|
||||
continue
|
||||
previous_flow_depth = flow_depth
|
||||
flow_references, flow_depth = _flow_uses_references(active_line, flow_depth)
|
||||
for reference in flow_references:
|
||||
yield line_number, reference
|
||||
if previous_flow_depth == 0 and flow_depth > 0:
|
||||
flow_start_line = line_number
|
||||
elif flow_depth == 0:
|
||||
flow_start_line = None
|
||||
if explicit_uses_line is not None:
|
||||
yield explicit_uses_line, "<missing explicit mapping value>"
|
||||
if flow_depth > 0:
|
||||
yield flow_start_line or 1, "<unterminated flow collection>"
|
||||
if block_uses_line is not None:
|
||||
yield block_uses_line, "\n".join(block_uses_content).strip()
|
||||
return workflow_yaml.iter_uses_references(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def validate_reference(reference: str) -> str | None:
|
||||
@@ -349,7 +34,11 @@ def validate_reference(reference: str) -> str | None:
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = list(sys.argv[1:] if argv is None else argv)
|
||||
paths = [Path(value) for value in args] if args else list(iter_workflow_files())
|
||||
paths = (
|
||||
[Path(value) for value in args]
|
||||
if args
|
||||
else [*workflow_yaml.iter_workflow_files(WORKFLOWS), *iter_action_files()]
|
||||
)
|
||||
violations = []
|
||||
for path in paths:
|
||||
for line_number, reference in iter_uses_references(path):
|
||||
|
||||
@@ -5,12 +5,13 @@ from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
from workflow_yaml import iter_uses_references, iter_workflow_files, scalar
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
WORKFLOWS = ROOT / ".github" / "workflows"
|
||||
CI_WORKFLOW = Path(".github/workflows/ci.yml")
|
||||
FULL_SHA = re.compile(r"[0-9a-f]{40}")
|
||||
USES_LINE = re.compile(r"^\s*(?:-\s+)?(?:uses|'uses'|\"uses\")\s*:\s*(.*?)\s*$")
|
||||
|
||||
|
||||
def _active_text(text: str) -> str:
|
||||
@@ -19,34 +20,6 @@ def _active_text(text: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _scalar(value: str) -> str:
|
||||
"""Remove an inline YAML comment and matching scalar quotes."""
|
||||
quote: str | None = None
|
||||
escaped = False
|
||||
end = len(value)
|
||||
for index, character in enumerate(value):
|
||||
if escaped:
|
||||
escaped = False
|
||||
continue
|
||||
if quote == '"' and character == "\\":
|
||||
escaped = True
|
||||
continue
|
||||
if character in ("'", '"'):
|
||||
if quote is None:
|
||||
quote = character
|
||||
elif quote == character:
|
||||
quote = None
|
||||
elif character == "#" and quote is None and (
|
||||
index == 0 or value[index - 1].isspace()
|
||||
):
|
||||
end = index
|
||||
break
|
||||
result = value[:end].strip()
|
||||
if len(result) >= 2 and result[0] == result[-1] and result[0] in ("'", '"'):
|
||||
return result[1:-1]
|
||||
return result
|
||||
|
||||
|
||||
def _has_trigger(text: str, event: str) -> bool:
|
||||
lines = text.splitlines()
|
||||
on_key = r"""(?:on|'on'|"on")"""
|
||||
@@ -61,7 +34,7 @@ def _has_trigger(text: str, event: str) -> bool:
|
||||
match = re.fullmatch(rf"{on_key}:\s*(.+?)\s*", line)
|
||||
if match is not None and re.search(
|
||||
rf"""(?:^|[\[{{,\s])['"]?{re.escape(event)}['"]?(?:$|[\]}},\s:])""",
|
||||
_scalar(match.group(1)),
|
||||
scalar(match.group(1)),
|
||||
):
|
||||
return True
|
||||
return False
|
||||
@@ -104,7 +77,7 @@ def _check_fail_open(path: Path, text: str) -> list[str]:
|
||||
r"""^\s*(?:-\s+)?(?:continue-on-error|'continue-on-error'|"continue-on-error")\s*:\s*(.*?)\s*$""",
|
||||
line,
|
||||
)
|
||||
if match is not None and _scalar(match.group(1)).lower() != "false":
|
||||
if match is not None and scalar(match.group(1)).lower() != "false":
|
||||
errors.append(f"{path}:{line_number}: continue-on-error must remain false")
|
||||
if re.search(r"\|\|\s*true(?:\s|$)", line):
|
||||
errors.append(f"{path}:{line_number}: command must not suppress failure with || true")
|
||||
@@ -121,26 +94,22 @@ def check_workflow(path: Path, text: str) -> list[str]:
|
||||
|
||||
pull_request = _has_trigger(active, "pull_request")
|
||||
lines = active.splitlines()
|
||||
for line_index, line in enumerate(lines):
|
||||
match = USES_LINE.match(line)
|
||||
if match is None:
|
||||
continue
|
||||
reference = _scalar(match.group(1))
|
||||
for line_number, reference in iter_uses_references(active):
|
||||
if reference.startswith("./"):
|
||||
continue
|
||||
action, separator, ref = reference.rpartition("@")
|
||||
if not separator or not action or FULL_SHA.fullmatch(ref) is None:
|
||||
errors.append(
|
||||
f"{path}:{line_index + 1}: external action must use a full commit SHA: {reference}"
|
||||
f"{path}:{line_number}: external action must use a full commit SHA: {reference}"
|
||||
)
|
||||
if pull_request and action == "actions/checkout":
|
||||
step = _step_block(lines, line_index)
|
||||
step = _step_block(lines, line_number - 1)
|
||||
if re.search(
|
||||
r"""(?mi)^\s+(?:persist-credentials|'persist-credentials'|"persist-credentials")\s*:\s*['"]?false['"]?\s*(?:#.*)?$""",
|
||||
step,
|
||||
) is None:
|
||||
errors.append(
|
||||
f"{path}:{line_index + 1}: pull-request checkout must discard GitHub credentials"
|
||||
f"{path}:{line_number}: pull-request checkout must discard GitHub credentials"
|
||||
)
|
||||
|
||||
if re.search(
|
||||
@@ -163,7 +132,7 @@ def check_workflow(path: Path, text: str) -> list[str]:
|
||||
|
||||
def main() -> int:
|
||||
errors: list[str] = []
|
||||
for path in sorted((*WORKFLOWS.glob("*.yml"), *WORKFLOWS.glob("*.yaml"))):
|
||||
for path in iter_workflow_files(WORKFLOWS):
|
||||
errors.extend(check_workflow(path.relative_to(ROOT), path.read_text(encoding="utf-8")))
|
||||
|
||||
if errors:
|
||||
|
||||
+1
-24
@@ -82,30 +82,7 @@ fi
|
||||
|
||||
# 4. Workflow and script regression guards
|
||||
section "workflow and script guards"
|
||||
if python3 scripts/check_build_workflow.py &&
|
||||
python3 scripts/test_check_build_workflow.py &&
|
||||
python3 scripts/check_apple_spm_locks.py &&
|
||||
python3 scripts/test_check_apple_spm_locks.py &&
|
||||
python3 scripts/check_workflow_security.py &&
|
||||
python3 scripts/test_check_workflow_security.py &&
|
||||
python3 scripts/check_workflow_action_pins.py &&
|
||||
python3 scripts/test_check_workflow_action_pins.py &&
|
||||
python3 scripts/check_container_image_pins.py &&
|
||||
python3 scripts/test_check_container_image_pins.py &&
|
||||
python3 scripts/verify_runtime_inputs.py &&
|
||||
python3 scripts/test_verify_runtime_inputs.py &&
|
||||
python3 scripts/test_fetch_tvos_engine.py &&
|
||||
python3 scripts/test_check_codegen.py &&
|
||||
python3 scripts/test_generate_relay_protocol.py &&
|
||||
python3 scripts/test_format_native.py &&
|
||||
python3 scripts/check_update_packages_workflow.py &&
|
||||
python3 scripts/test_pubspec_version.py &&
|
||||
python3 scripts/test_clean_translations.py &&
|
||||
python3 scripts/test_run_maestro.py &&
|
||||
python3 scripts/test_maestro_flow_contracts.py &&
|
||||
python3 scripts/test_maestro_jellyfin_proxy.py &&
|
||||
python3 scripts/test_maestro_real_jellyfin.py &&
|
||||
python3 scripts/test_check_icon_consistency.py; then
|
||||
if bash scripts/ci_guard_checks.sh; then
|
||||
ok "workflow and script guards passed"
|
||||
else
|
||||
fail "workflow or script guard failed"
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# Workflow and script regression guards.
|
||||
#
|
||||
# Single source of truth for the guard roster, shared by the "Verify workflow
|
||||
# and script guards" step in .github/workflows/ci.yml and section 4 of
|
||||
# scripts/ci_checks.sh. The checkers are named explicitly because a few of them
|
||||
# belong to other jobs (check_bun_audit.py needs Bun, check_codegen.py runs via
|
||||
# codegen.sh), but their regression tests are discovered by glob so a newly
|
||||
# added scripts/test_*.py is picked up automatically instead of having to be
|
||||
# remembered in two places.
|
||||
set -euo pipefail
|
||||
shopt -s nullglob
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
for checker in \
|
||||
scripts/check_build_workflow.py \
|
||||
scripts/check_apple_spm_locks.py \
|
||||
scripts/verify_runtime_inputs.py \
|
||||
scripts/check_workflow_security.py \
|
||||
scripts/check_workflow_action_pins.py \
|
||||
scripts/check_container_image_pins.py \
|
||||
scripts/check_update_packages_workflow.py; do
|
||||
python3 "$checker"
|
||||
done
|
||||
|
||||
for guard_test in scripts/test_*.py; do
|
||||
python3 "$guard_test"
|
||||
done
|
||||
@@ -0,0 +1,358 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared YAML scanning for the GitHub Actions guard scripts.
|
||||
|
||||
The guards deliberately avoid a YAML dependency, so the scalar plumbing and the
|
||||
`uses:` scanner live here once rather than being re-implemented, with differing
|
||||
rigor, in every checker.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
MAPPING_RE = re.compile(
|
||||
r"""^\s*(?:-\s*)?(?P<key>uses|'(?:''|[^'])*'|"(?:\\.|[^"\\])*")\s*:\s*(?P<value>.*?)\s*$"""
|
||||
)
|
||||
EXPLICIT_KEY_RE = re.compile(
|
||||
r"""^\s*(?:-\s*)?\?\s*(?P<key>uses|'(?:''|[^'])*'|"(?:\\.|[^"\\])*")\s*$"""
|
||||
)
|
||||
EXPLICIT_VALUE_RE = re.compile(r"^\s*:\s*(?P<value>.*?)\s*$")
|
||||
BLOCK_SCALAR_RE = re.compile(r":\s*[|>](?:[1-9][+-]?|[+-][1-9]?)?\s*(?:#.*)?$")
|
||||
BLOCK_SCALAR_VALUE_RE = re.compile(r"^[|>](?:[1-9][+-]?|[+-][1-9]?)?$")
|
||||
YAML_DOUBLE_ESCAPES = {
|
||||
"0": "\0",
|
||||
"a": "\a",
|
||||
"b": "\b",
|
||||
"t": "\t",
|
||||
"\t": "\t",
|
||||
"n": "\n",
|
||||
"v": "\v",
|
||||
"f": "\f",
|
||||
"r": "\r",
|
||||
"e": "\x1b",
|
||||
" ": " ",
|
||||
'"': '"',
|
||||
"/": "/",
|
||||
"\\": "\\",
|
||||
"N": "\u0085",
|
||||
"_": "\u00a0",
|
||||
"L": "\u2028",
|
||||
"P": "\u2029",
|
||||
}
|
||||
|
||||
|
||||
def iter_workflow_files(directory: Path):
|
||||
yield from sorted((*directory.glob("*.yml"), *directory.glob("*.yaml")))
|
||||
|
||||
|
||||
def job_block(text: str, name: str) -> str:
|
||||
"""Return the YAML block of a top-level job, or "" when it is absent."""
|
||||
match = re.search(
|
||||
rf"(?ms)^ {re.escape(name)}:\n(.*?)(?=^ [a-zA-Z0-9_-]+:\n|\Z)", text
|
||||
)
|
||||
return match.group(0) if match else ""
|
||||
|
||||
|
||||
def strip_comment(value: str) -> str:
|
||||
quote = None
|
||||
escaped = False
|
||||
for index, char in enumerate(value):
|
||||
if escaped:
|
||||
escaped = False
|
||||
continue
|
||||
if char == "\\" and quote == '"':
|
||||
escaped = True
|
||||
continue
|
||||
if char in ("'", '"'):
|
||||
if quote is None:
|
||||
quote = char
|
||||
elif quote == char:
|
||||
quote = None
|
||||
continue
|
||||
if char == "#" and quote is None and (index == 0 or value[index - 1].isspace()):
|
||||
return value[:index].rstrip()
|
||||
return value.rstrip()
|
||||
|
||||
|
||||
def _decode_quoted_yaml_string(value: str) -> str | None:
|
||||
if len(value) < 2 or value[0] != value[-1] or value[0] not in ("'", '"'):
|
||||
return None
|
||||
if value[0] == "'":
|
||||
return value[1:-1].replace("''", "'")
|
||||
|
||||
decoded = []
|
||||
index = 1
|
||||
end = len(value) - 1
|
||||
while index < end:
|
||||
char = value[index]
|
||||
if char != "\\":
|
||||
decoded.append(char)
|
||||
index += 1
|
||||
continue
|
||||
index += 1
|
||||
if index >= end:
|
||||
return None
|
||||
escape = value[index]
|
||||
if escape in YAML_DOUBLE_ESCAPES:
|
||||
decoded.append(YAML_DOUBLE_ESCAPES[escape])
|
||||
index += 1
|
||||
continue
|
||||
width = {"x": 2, "u": 4, "U": 8}.get(escape)
|
||||
if width is None or index + width >= end:
|
||||
return None
|
||||
digits = value[index + 1 : index + 1 + width]
|
||||
if not re.fullmatch(rf"[0-9a-fA-F]{{{width}}}", digits):
|
||||
return None
|
||||
try:
|
||||
decoded.append(chr(int(digits, 16)))
|
||||
except ValueError:
|
||||
return None
|
||||
index += width + 1
|
||||
return "".join(decoded)
|
||||
|
||||
|
||||
def unquote(value: str) -> str:
|
||||
decoded = _decode_quoted_yaml_string(value)
|
||||
return value if decoded is None else decoded
|
||||
|
||||
|
||||
def scalar(value: str) -> str:
|
||||
"""Read a single-line YAML scalar: drop an inline comment and its quotes."""
|
||||
return unquote(strip_comment(value).strip())
|
||||
|
||||
|
||||
def _flow_value(line: str, start: int, mapping_depth: int) -> str:
|
||||
index = start
|
||||
quote = None
|
||||
escaped = False
|
||||
depth = mapping_depth
|
||||
while index < len(line):
|
||||
char = line[index]
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\" and quote == '"':
|
||||
escaped = True
|
||||
elif quote is not None:
|
||||
if char == quote:
|
||||
quote = None
|
||||
elif char in ("'", '"'):
|
||||
quote = char
|
||||
elif char in ("{", "["):
|
||||
depth += 1
|
||||
elif char in ("}", "]"):
|
||||
if depth == mapping_depth:
|
||||
break
|
||||
depth -= 1
|
||||
elif char == "," and depth == mapping_depth:
|
||||
break
|
||||
index += 1
|
||||
return unquote(line[start:index].strip())
|
||||
|
||||
|
||||
def _has_unsupported_block_mapping_key(line: str) -> bool:
|
||||
candidate = line.lstrip()
|
||||
if candidate.startswith("-") and not candidate.startswith("---"):
|
||||
candidate = candidate[1:].lstrip()
|
||||
if not candidate:
|
||||
return False
|
||||
if candidate[0] in "!&*":
|
||||
return True
|
||||
if candidate[0] not in ("'", '"'):
|
||||
return False
|
||||
|
||||
quote = candidate[0]
|
||||
escaped = False
|
||||
index = 1
|
||||
while index < len(candidate):
|
||||
char = candidate[index]
|
||||
if quote == "'" and char == "'" and index + 1 < len(candidate) and candidate[index + 1] == "'":
|
||||
index += 2
|
||||
continue
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif quote == '"' and char == "\\":
|
||||
escaped = True
|
||||
elif char == quote:
|
||||
return False
|
||||
index += 1
|
||||
return True
|
||||
|
||||
|
||||
def _flow_uses_references(line: str, initial_depth: int) -> tuple[list[str], int]:
|
||||
references = []
|
||||
depth = initial_depth
|
||||
index = 0
|
||||
while index < len(line):
|
||||
char = line[index]
|
||||
if char in ("'", '"'):
|
||||
quote = char
|
||||
escaped = False
|
||||
end = index + 1
|
||||
while end < len(line):
|
||||
quoted_char = line[end]
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif quoted_char == "\\" and quote == '"':
|
||||
escaped = True
|
||||
elif quoted_char == quote:
|
||||
break
|
||||
end += 1
|
||||
if end >= len(line):
|
||||
if depth > 0:
|
||||
references.append("<unsupported multiline flow scalar>")
|
||||
return references, depth
|
||||
key = _decode_quoted_yaml_string(line[index : end + 1])
|
||||
after_key = end + 1
|
||||
while after_key < len(line) and line[after_key].isspace():
|
||||
after_key += 1
|
||||
if depth > 0 and after_key < len(line) and line[after_key] == ":":
|
||||
if key == "uses":
|
||||
references.append(_flow_value(line, after_key + 1, depth))
|
||||
elif key is None:
|
||||
references.append("<unsupported quoted flow mapping key>")
|
||||
index = end + 1
|
||||
continue
|
||||
if line.startswith("${{", index):
|
||||
expression_end = line.find("}}", index + 3)
|
||||
if expression_end < 0:
|
||||
references.append("<unterminated GitHub expression>")
|
||||
return references, depth
|
||||
index = expression_end + 2
|
||||
continue
|
||||
if char in ("{", "["):
|
||||
depth += 1
|
||||
index += 1
|
||||
continue
|
||||
if char in ("}", "]"):
|
||||
depth = max(0, depth - 1)
|
||||
index += 1
|
||||
continue
|
||||
if depth > 0 and char == "?":
|
||||
references.append("<unsupported explicit flow mapping>")
|
||||
index += 1
|
||||
continue
|
||||
if depth > 0 and char in "!&*":
|
||||
references.append("<unsupported tagged, anchored, or aliased flow mapping>")
|
||||
index += 1
|
||||
continue
|
||||
if depth > 0 and (char.isalpha() or char == "_"):
|
||||
end = index + 1
|
||||
while end < len(line) and (line[end].isalnum() or line[end] in "_-"):
|
||||
end += 1
|
||||
after_key = end
|
||||
while after_key < len(line) and line[after_key].isspace():
|
||||
after_key += 1
|
||||
if line[index:end] == "uses" and after_key < len(line) and line[after_key] == ":":
|
||||
references.append(_flow_value(line, after_key + 1, depth))
|
||||
index = end
|
||||
continue
|
||||
index += 1
|
||||
return references, depth
|
||||
|
||||
|
||||
def iter_uses_references(text: str):
|
||||
"""Yield (line number, reference) for every `uses:` value in a workflow.
|
||||
|
||||
Constructs the scanner cannot resolve are yielded as `<...>` placeholders so
|
||||
that callers fail closed rather than silently skipping an unpinned action.
|
||||
"""
|
||||
block_parent_indent = None
|
||||
block_content_indent = None
|
||||
block_uses_line = None
|
||||
block_uses_content: list[str] = []
|
||||
explicit_uses_line = None
|
||||
flow_start_line = None
|
||||
flow_depth = 0
|
||||
for line_number, raw_line in enumerate(text.splitlines(), 1):
|
||||
stripped = raw_line.lstrip()
|
||||
indent = len(raw_line) - len(stripped)
|
||||
if block_parent_indent is not None:
|
||||
if not stripped:
|
||||
if block_uses_line is not None:
|
||||
block_uses_content.append("")
|
||||
continue
|
||||
if indent <= block_parent_indent:
|
||||
if block_uses_line is not None:
|
||||
yield block_uses_line, "\n".join(block_uses_content).strip()
|
||||
block_parent_indent = None
|
||||
block_content_indent = None
|
||||
block_uses_line = None
|
||||
block_uses_content = []
|
||||
elif block_content_indent is None:
|
||||
block_content_indent = indent
|
||||
if block_uses_line is not None:
|
||||
block_uses_content.append(raw_line[block_content_indent:])
|
||||
continue
|
||||
elif indent >= block_content_indent:
|
||||
if block_uses_line is not None:
|
||||
block_uses_content.append(raw_line[block_content_indent:])
|
||||
continue
|
||||
else:
|
||||
if block_uses_line is not None:
|
||||
yield block_uses_line, "\n".join(block_uses_content).strip()
|
||||
block_parent_indent = None
|
||||
block_content_indent = None
|
||||
block_uses_line = None
|
||||
block_uses_content = []
|
||||
if stripped.startswith("#") or not stripped:
|
||||
continue
|
||||
active_line = strip_comment(raw_line)
|
||||
if explicit_uses_line is not None:
|
||||
explicit_value = EXPLICIT_VALUE_RE.match(active_line)
|
||||
if explicit_value is None:
|
||||
yield explicit_uses_line, "<missing explicit mapping value>"
|
||||
else:
|
||||
value = explicit_value.group("value").strip()
|
||||
if BLOCK_SCALAR_VALUE_RE.fullmatch(value):
|
||||
block_parent_indent = indent
|
||||
block_content_indent = None
|
||||
block_uses_line = explicit_uses_line
|
||||
block_uses_content = []
|
||||
else:
|
||||
yield explicit_uses_line, unquote(value)
|
||||
explicit_uses_line = None
|
||||
continue
|
||||
explicit_uses_line = None
|
||||
explicit_key = EXPLICIT_KEY_RE.match(active_line)
|
||||
if explicit_key:
|
||||
if unquote(explicit_key.group("key")) == "uses":
|
||||
explicit_uses_line = line_number
|
||||
continue
|
||||
if re.match(r"^\s*(?:-\s*)?\?", active_line):
|
||||
yield line_number, "<unsupported explicit mapping key>"
|
||||
continue
|
||||
if _has_unsupported_block_mapping_key(active_line):
|
||||
yield line_number, "<unsupported multiline, tagged, anchored, or aliased mapping key>"
|
||||
continue
|
||||
match = MAPPING_RE.match(active_line) if flow_depth == 0 else None
|
||||
if match:
|
||||
key = unquote(match.group("key"))
|
||||
value = match.group("value").strip()
|
||||
if BLOCK_SCALAR_VALUE_RE.fullmatch(value):
|
||||
block_parent_indent = indent
|
||||
block_content_indent = None
|
||||
if key == "uses":
|
||||
block_uses_line = line_number
|
||||
block_uses_content = []
|
||||
continue
|
||||
if key == "uses":
|
||||
yield line_number, unquote(value)
|
||||
if BLOCK_SCALAR_RE.search(raw_line):
|
||||
block_parent_indent = indent
|
||||
block_content_indent = None
|
||||
continue
|
||||
previous_flow_depth = flow_depth
|
||||
flow_references, flow_depth = _flow_uses_references(active_line, flow_depth)
|
||||
for reference in flow_references:
|
||||
yield line_number, reference
|
||||
if previous_flow_depth == 0 and flow_depth > 0:
|
||||
flow_start_line = line_number
|
||||
elif flow_depth == 0:
|
||||
flow_start_line = None
|
||||
if explicit_uses_line is not None:
|
||||
yield explicit_uses_line, "<missing explicit mapping value>"
|
||||
if flow_depth > 0:
|
||||
yield flow_start_line or 1, "<unterminated flow collection>"
|
||||
if block_uses_line is not None:
|
||||
yield block_uses_line, "\n".join(block_uses_content).strip()
|
||||
Reference in New Issue
Block a user