# Any copyright is dedicated to the public domain.
# http://creativecommons.org/publicdomain/zero/1.0/
import hashlib
import importlib.util
import sys
from pathlib import Path
import pytest
from mozunit import main
SCRIPT = (
Path(__file__).resolve().parent.parent
/ "scripts"
/ "misc"
/ "android-gradle-dependencies"
/ "verify_dependencies.py"
)
spec = importlib.util.spec_from_file_location("verify_dependencies", SCRIPT)
verify_dependencies = importlib.util.module_from_spec(spec)
spec.loader.exec_module(verify_dependencies)
def digest(content):
return hashlib.sha256(content).hexdigest()
def write_inventory(directory, name, components):
"""Write an inventory in the shape `--write-verification-metadata` does.
`components` maps "group:name:version" to {filename: content}, and the
recorded checksum is that of the content, so a test makes a component
corrupt or missing by writing something else, or nothing, to the repository.
"""
lines = [
'',
'',
" ",
" true",
" ",
" ",
]
for coordinates, files in components.items():
group, module, version = coordinates.split(":")
lines.append(
f' '
)
for filename, content in files.items():
lines.append(f' ')
lines.append(
f' '
)
lines.append(" ")
lines.append(" ")
lines += [" ", ""]
directory.mkdir(parents=True, exist_ok=True)
inventory = directory / f"{name}.xml"
inventory.write_text("\n".join(lines), encoding="utf-8")
return inventory
def write_raw_inventory(directory, name, body):
"""An inventory written by hand, for shapes the helpers cannot express."""
directory.mkdir(parents=True, exist_ok=True)
inventory = directory / f"{name}.xml"
inventory.write_text(
''
''
f"{body}",
encoding="utf-8",
)
return inventory
def write_inventory_without_checksums(directory, name, components):
"""An inventory whose artifacts carry no , which verify() can only
check for existence."""
body = ""
for coordinates, files in components.items():
group, module, version = coordinates.split(":")
artifacts = "".join(f'' for filename in files)
body += (
f''
f"{artifacts}"
)
return write_raw_inventory(directory, name, body)
def populate(repository, components):
"""Lay components out in a repository the way a Maven repository does."""
for coordinates, files in components.items():
group, module, version = coordinates.split(":")
directory = repository.joinpath(*group.split("."), module, version)
directory.mkdir(parents=True, exist_ok=True)
for filename, content in files.items():
(directory / filename).write_bytes(content)
return repository
def check(inventories, repositories, unproxied=()):
inventory, _, _ = verify_dependencies.merge_inventories(sorted(inventories))
return verify_dependencies.verify(inventory, repositories, unproxied)
def test_complete_tree(tmp_path):
components = {
"com.example:widget:1.0": {
"widget-1.0.jar": b"jar",
"widget-1.0.pom": b"pom",
},
"androidx.core:core:1.2.3": {"core-1.2.3.aar": b"aar"},
}
inventory = write_inventory(tmp_path / "inventories", "top-level", components)
repository = populate(tmp_path / "central", components)
result = check([inventory], [repository])
assert (result.problems, result.unpackaged) == ([], [])
assert (result.artifacts, result.redirected) == (3, 0)
def test_component_split_across_repositories(tmp_path):
"""Downstream tasks are handed every repository, so either will do."""
inventory = write_inventory(
tmp_path / "inventories",
"top-level",
{
"com.example:widget:1.0": {"widget-1.0.jar": b"jar"},
"androidx.core:core:1.2.3": {"core-1.2.3.aar": b"aar"},
},
)
central = populate(
tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}}
)
google = populate(
tmp_path / "google", {"androidx.core:core:1.2.3": {"core-1.2.3.aar": b"aar"}}
)
result = check([inventory], [central, google])
assert result.problems == []
assert (result.artifacts, result.redirected) == (2, 0)
def test_inventories_are_merged(tmp_path):
"""Each pass covers one Gradle build; a shared component is packaged once."""
inventories = tmp_path / "inventories"
shared = {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}}
first = write_inventory(
inventories,
"top-level",
{**shared, "com.example:only-top:1.0": {"only-top-1.0.jar": b"a"}},
)
second = write_inventory(
inventories,
"fenix",
{**shared, "com.example:only-fenix:1.0": {"only-fenix-1.0.jar": b"b"}},
)
repository = populate(
tmp_path / "central",
{
**shared,
"com.example:only-top:1.0": {"only-top-1.0.jar": b"a"},
"com.example:only-fenix:1.0": {"only-fenix-1.0.jar": b"b"},
},
)
merged, counts, anomalies = verify_dependencies.merge_inventories([first, second])
assert len(merged) == 3
assert sorted(counts.values()) == [2, 2]
assert anomalies == []
assert check([first, second], [repository]).problems == []
def test_file_published_by_another_module(tmp_path):
"""A component's metadata can name a file a different module publishes.
Gradle records `guava-33.3.1-jre.jar` under `guava:33.3.1-android`, and
Kotlin Multiplatform modules redirect to a platform-specific module. The
bytes are packaged, just not where this component's coordinates point.
"""
inventory = write_inventory(
tmp_path / "inventories",
"top-level",
{
"com.google.guava:guava:33.3.1-android": {
"guava-33.3.1-android.jar": b"android",
"guava-33.3.1-jre.jar": b"jre",
}
},
)
repository = populate(
tmp_path / "central",
{
"com.google.guava:guava:33.3.1-android": {
"guava-33.3.1-android.jar": b"android"
},
"com.google.guava:guava:33.3.1-jre": {"guava-33.3.1-jre.jar": b"jre"},
},
)
result = check([inventory], [repository])
assert result.problems == []
assert (result.artifacts, result.redirected) == (1, 1)
def test_file_published_under_another_name(tmp_path):
"""The recorded name need not be the name on disk either, so fall back to
matching the contents within the component's own directory."""
inventory = write_inventory(
tmp_path / "inventories",
"top-level",
{"org.example.kmp:library:1.0": {"library-1.0.jar": b"bytes"}},
)
repository = populate(
tmp_path / "central",
{"org.example.kmp:library:1.0": {"library-jvm-1.0.jar": b"bytes"}},
)
result = check([inventory], [repository])
assert result.problems == []
assert (result.artifacts, result.redirected) == (0, 1)
def test_missing_file(tmp_path):
"""The intermittent fault: the component is there, one of its files isn't."""
inventory = write_inventory(
tmp_path / "inventories",
"top-level",
{
"com.example:widget:1.0": {
"widget-1.0.jar": b"jar",
"widget-1.0.pom": b"pom",
}
},
)
repository = populate(
tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.pom": b"pom"}}
)
problems = check([inventory], [repository]).problems
assert problems == [("missing", ("com.example", "widget", "1.0"), "widget-1.0.jar")]
def test_corrupt_file(tmp_path):
inventory = write_inventory(
tmp_path / "inventories",
"top-level",
{"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}},
)
repository = populate(
tmp_path / "central",
{"com.example:widget:1.0": {"widget-1.0.jar": b"truncated"}},
)
problems = check([inventory], [repository]).problems
assert len(problems) == 1
kind, component, detail = problems[0]
assert (kind, component) == ("corrupt", ("com.example", "widget", "1.0"))
assert detail.endswith("widget-1.0.jar")
def test_absent_component(tmp_path):
inventory = write_inventory(
tmp_path / "inventories",
"top-level",
{
"com.example:widget:1.0": {"widget-1.0.jar": b"jar"},
"com.example:absent:2.0": {"absent-2.0.jar": b"jar"},
},
)
repository = populate(
tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}}
)
problems = check([inventory], [repository]).problems
# If this differs on some platform, fix the reporting, not the expectation.
assert problems == [
("absent", ("com.example", "absent", "2.0"), "com/example/absent/2.0")
]
def test_unrecorded_files_are_ignored(tmp_path):
"""Nexus also stores what `mach build` resolved, which no pass enumerates."""
inventory = write_inventory(
tmp_path / "inventories",
"top-level",
{"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}},
)
repository = populate(
tmp_path / "central",
{
"com.example:widget:1.0": {
"widget-1.0.jar": b"jar",
"widget-1.0.pom": b"pom",
},
"com.example:unrecorded:1.0": {"unrecorded-1.0.jar": b"jar"},
},
)
assert check([inventory], [repository]).problems == []
@pytest.mark.parametrize(
"inventory,expected",
[
(None, "no inventories in"),
("", "list no components at all"),
],
ids=["no-inventories", "no-components"],
)
def test_main_rejects_an_unusable_inventory(
tmp_path, monkeypatch, capsys, inventory, expected
):
"""A pass that did not run must not read as a cache with nothing missing."""
inventories = tmp_path / "inventories"
inventories.mkdir()
if inventory is not None:
(inventories / "top-level.xml").write_text(inventory, encoding="utf-8")
repository = tmp_path / "central"
repository.mkdir()
monkeypatch.setattr(
sys,
"argv",
["verify_dependencies.py", "--inventories", str(inventories), str(repository)],
)
assert verify_dependencies.main() == 1
assert expected in capsys.readouterr().out
def test_main_reports_a_complete_tree(tmp_path, monkeypatch, capsys):
components = {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}}
write_inventory(tmp_path / "inventories", "top-level", components)
populate(tmp_path / "central", components)
monkeypatch.setattr(
sys,
"argv",
[
"verify_dependencies.py",
"--inventories",
str(tmp_path / "inventories"),
str(tmp_path / "central"),
],
)
assert verify_dependencies.main() == 0
out = capsys.readouterr().out
assert "top-level: 1 components" in out
assert "verified 1 artifacts across 1 of 1 components" in out
def test_main_rejects_a_missing_repository(tmp_path, monkeypatch, capsys):
"""A repository that moved must not read as its components being absent."""
components = {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}}
write_inventory(tmp_path / "inventories", "top-level", components)
populate(tmp_path / "central", components)
monkeypatch.setattr(
sys,
"argv",
[
"verify_dependencies.py",
"--inventories",
str(tmp_path / "inventories"),
str(tmp_path / "central"),
str(tmp_path / "gone"),
],
)
assert verify_dependencies.main() == 1
assert "not a packaged repository" in capsys.readouterr().out
def test_unproxied_absence_is_expected(tmp_path):
"""GeckoView is built by this task, not cached, so it is absent by design."""
inventory = write_inventory(
tmp_path / "inventories",
"fenix",
{
"com.example:widget:1.0": {"widget-1.0.jar": b"jar"},
"org.mozilla.geckoview:geckoview-default-omni:1.2.3": {
"geckoview-default-omni-1.2.3.aar": b"built here"
},
},
)
repository = populate(
tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}}
)
result = check([inventory], [repository], ("org/mozilla/geckoview",))
assert result.problems == []
assert result.unpackaged == [
("org.mozilla.geckoview", "geckoview-default-omni", "1.2.3")
]
# Without the prefix it is a fault, and glean is not covered by it either.
problems = check([inventory], [repository]).problems
assert [kind for kind, _, _ in problems] == ["absent"]
def test_unproxied_does_not_exempt_a_longer_named_sibling(tmp_path):
"""org/mozilla/geckoview must not cover org/mozilla/geckoviewextra."""
inventory = write_inventory(
tmp_path / "inventories",
"fenix",
{"org.mozilla.geckoviewextra:widget:1.0": {"widget-1.0.jar": b"jar"}},
)
repository = tmp_path / "central"
repository.mkdir()
result = check([inventory], [repository], ("org/mozilla/geckoview",))
assert result.unpackaged == []
assert [kind for kind, _, _ in result.problems] == ["absent"]
# The prefix itself, and a parent of it, do exempt.
for prefix in ("org/mozilla/geckoviewextra", "org/mozilla"):
result = check([inventory], [repository], (prefix,))
assert (result.problems, len(result.unpackaged)) == ([], 1), prefix
def test_report_separates_the_kinds_and_their_advice(tmp_path, capsys):
"""Absent and incompletely packaged want different things from the reader."""
verify_dependencies.report([
("absent", ("org.example", "gone", "1.0"), "org/example/gone/1.0"),
("missing", ("com.example", "widget", "1.0"), "widget-1.0.jar"),
("corrupt", ("com.example", "other", "2.0"), "/packaged/other-2.0.jar"),
])
out = capsys.readouterr().out
# Treeherder's log parser surfaces the "FATAL ERROR" line and nothing else,
# so the tally has to be on it. See treeherder/log_parser/parsers.py.
fatal = [line for line in out.splitlines() if "FATAL ERROR" in line]
assert fatal == [
"FATAL ERROR: the packaged dependency cache is incomplete: "
"1 component absent, 1 file missing, 1 file corrupt."
]
assert "1 component resolved by an enumeration pass" in out
assert " absent: org.example:gone:1.0 (looked for org/example/gone/1.0)" in out
assert "--unproxied" in out
assert "1 file resolved from a component that is packaged" in out
assert " missing: com.example:widget:1.0 -> widget-1.0.jar" in out
assert "Re-running the task usually produces a complete one." in out
assert "1 packaged file whose contents do not match" in out
assert " corrupt: /packaged/other-2.0.jar" in out
@pytest.mark.parametrize("prefix", ["", "."], ids=["empty", "dot"])
def test_main_rejects_an_empty_unproxied_prefix(tmp_path, monkeypatch, capsys, prefix):
"""An empty prefix is every component's parent, so it would excuse the tree."""
components = {"com.example:absent:1.0": {"absent-1.0.jar": b"jar"}}
write_inventory(tmp_path / "inventories", "top-level", components)
repository = tmp_path / "central"
repository.mkdir()
monkeypatch.setattr(
sys,
"argv",
[
"verify_dependencies.py",
"--inventories",
str(tmp_path / "inventories"),
"--unproxied",
prefix,
str(repository),
],
)
assert verify_dependencies.main() == 1
out = capsys.readouterr().out
assert "--unproxied needs a path prefix" in out
# It must fail on the flag, not sail past it and exempt the missing component.
assert "not packaged, as expected" not in out
def test_conflicting_checksums_are_reported_but_not_fatal(
tmp_path, monkeypatch, capsys
):
"""Only one file can be packaged and downstream resolves it happily, so the
disagreement is worth saying without failing the task."""
inventories = tmp_path / "inventories"
write_inventory(
inventories, "top-level", {"com.example:widget:1.0": {"widget-1.0.jar": b"one"}}
)
write_inventory(
inventories, "fenix", {"com.example:widget:1.0": {"widget-1.0.jar": b"two"}}
)
populate(
tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"two"}}
)
merged, _, anomalies = verify_dependencies.merge_inventories(
sorted(inventories.glob("*.xml"))
)
assert merged[("com.example", "widget", "1.0")]["widget-1.0.jar"] == {
digest(b"one"),
digest(b"two"),
}
assert anomalies == [
"com.example:widget:1.0 -> widget-1.0.jar is recorded with more than "
"one checksum; any of them is accepted"
]
monkeypatch.setattr(
sys,
"argv",
[
"verify_dependencies.py",
"--inventories",
str(inventories),
str(tmp_path / "central"),
],
)
assert verify_dependencies.main() == 0
out = capsys.readouterr().out
assert "WARNING: com.example:widget:1.0 -> widget-1.0.jar" in out
assert "FATAL" not in out
def test_a_pass_without_a_checksum_does_not_erase_one(tmp_path):
"""Otherwise the artifact stops being integrity-checked, and the redirect
path reports it missing because it has nothing to match on."""
inventories = tmp_path / "inventories"
component = {
"com.google.guava:guava:33.3.1-android": {"guava-33.3.1-jre.jar": b"jre"}
}
# The checksumless pass must sort last, so that it is the one that would
# overwrite: merge_inventories reads them in name order.
write_inventory(inventories, "android-components", component)
write_inventory_without_checksums(inventories, "top-level", component)
# The -android component is packaged, but the file it records lives under
# the module that publishes it, so only the checksum finds it.
repository = populate(
tmp_path / "central",
{
"com.google.guava:guava:33.3.1-android": {
"guava-33.3.1-android.jar": b"android"
},
"com.google.guava:guava:33.3.1-jre": {"guava-33.3.1-jre.jar": b"jre"},
},
)
merged, _, anomalies = verify_dependencies.merge_inventories(
sorted(inventories.glob("*.xml"))
)
assert anomalies == []
# The real checksum survives alongside the empty record, so the redirect
# still has something to match on.
assert merged[("com.google.guava", "guava", "33.3.1-android")] == {
"guava-33.3.1-jre.jar": {None, digest(b"jre")}
}
result = check(sorted(inventories.glob("*.xml")), [repository])
assert result.problems == []
assert (result.artifacts, result.redirected) == (0, 1)
def test_an_artifact_no_pass_checksummed_is_still_checked_for_existence(tmp_path):
"""Dropping it from the merge entirely would stop it being checked at all."""
inventories = tmp_path / "inventories"
write_inventory_without_checksums(
inventories, "top-level", {"com.example:widget:1.0": {"widget-1.0.jar": None}}
)
repository = populate(tmp_path / "central", {"com.example:widget:1.0": {}})
merged, _, _ = verify_dependencies.merge_inventories(
sorted(inventories.glob("*.xml"))
)
assert merged == {("com.example", "widget", "1.0"): {"widget-1.0.jar": {None}}}
problems = check(sorted(inventories.glob("*.xml")), [repository]).problems
assert problems == [("missing", ("com.example", "widget", "1.0"), "widget-1.0.jar")]
def test_one_artifact_recorded_twice_in_one_inventory(tmp_path):
"""Same disagreement as across passes, and the same non-fatal handling."""
inventory = write_raw_inventory(
tmp_path / "inventories",
"top-level",
''
f''
f''
"",
)
repository = populate(
tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"one"}}
)
merged, _, anomalies = verify_dependencies.merge_inventories([inventory])
assert merged[("com.example", "widget", "1.0")]["widget-1.0.jar"] == {
digest(b"one"),
digest(b"two"),
}
assert len(anomalies) == 1 and "more than one checksum" in anomalies[0]
# Either recorded file is accepted, so the packaged one is not "corrupt".
assert check([inventory], [repository]).problems == []
@pytest.mark.parametrize(
"body,expected",
[
(
''
'',
"a component has no coordinates",
),
(
''
'',
"records an artifact with no name",
),
],
ids=["no-coordinates", "no-artifact-name"],
)
def test_malformed_entries_are_reported_not_fatal(tmp_path, body, expected):
"""Skipping one entry checks less, but does not make the cache unusable."""
inventory = write_raw_inventory(tmp_path / "inventories", "top-level", body)
merged, _, anomalies = verify_dependencies.merge_inventories([inventory])
assert len(anomalies) == 1 and expected in anomalies[0]
# A skipped entry leaves nothing behind for verify() to trip over.
assert verify_dependencies.verify(merged, [tmp_path]).problems == []
def test_an_exempt_component_stays_exempt_when_partly_packaged(tmp_path):
"""Whether this task packages a component is a property of its coordinates.
A directory left behind at those coordinates must not turn the exemption
into a fatal, but is worth saying in case the exemption is stale."""
inventory = write_inventory(
tmp_path / "inventories",
"fenix",
{
"org.mozilla.geckoview:geckoview-default-omni:1.2.3": {
"geckoview-default-omni-1.2.3.aar": b"built here"
}
},
)
# The coordinates exist on disk, but without the file that was recorded.
repository = populate(
tmp_path / "central",
{"org.mozilla.geckoview:geckoview-default-omni:1.2.3": {"leftover.txt": b"x"}},
)
result = check([inventory], [repository], ("org/mozilla/geckoview",))
assert result.problems == []
assert result.unpackaged == [
("org.mozilla.geckoview", "geckoview-default-omni", "1.2.3")
]
assert len(result.anomalies) == 1
assert "exempt from packaging but is partly packaged" in result.anomalies[0]
def test_either_recorded_checksum_is_accepted(tmp_path):
"""The packaged file can match whichever record, not just the last one."""
inventories = tmp_path / "inventories"
write_inventory(
inventories,
"android-components",
{"com.example:widget:1.0": {"w-1.0.jar": b"one"}},
)
write_inventory(
inventories, "top-level", {"com.example:widget:1.0": {"w-1.0.jar": b"two"}}
)
# Matches the *first* record; the naive implementation kept only the last.
repository = populate(
tmp_path / "central", {"com.example:widget:1.0": {"w-1.0.jar": b"one"}}
)
result = check(sorted(inventories.glob("*.xml")), [repository])
assert result.problems == []
assert result.artifacts == 1
def test_unproxied_takes_more_than_one_prefix(tmp_path):
"""The flag is repeatable, so more than one exemption has to work at once."""
inventory = write_inventory(
tmp_path / "inventories",
"fenix",
{
"org.mozilla.geckoview:gv:1.0": {"gv-1.0.aar": b"a"},
"com.example.local:built:2.0": {"built-2.0.jar": b"b"},
"com.example:packaged:3.0": {"packaged-3.0.jar": b"c"},
},
)
repository = populate(
tmp_path / "central", {"com.example:packaged:3.0": {"packaged-3.0.jar": b"c"}}
)
result = check(
[inventory], [repository], ("org/mozilla/geckoview", "com/example/local")
)
assert result.problems == []
assert sorted(result.unpackaged) == [
("com.example.local", "built", "2.0"),
("org.mozilla.geckoview", "gv", "1.0"),
]
# Only the named prefixes are exempt; a sibling under com/example is not.
assert result.artifacts == 1
def test_only_component_elements_are_read(tmp_path):
"""Matched on the local name, so a nested element whose name merely ends in
"component" is not mistaken for one."""
inventory = write_raw_inventory(
tmp_path / "inventories",
"top-level",
''
f''
""
''
f''
"",
)
components, anomalies = verify_dependencies.parse_inventory(inventory)
assert list(components) == [("com.example", "real", "1.0")]
assert anomalies == []
if __name__ == "__main__":
main()