# Any copyright is dedicated to the public domain. # http://creativecommons.org/publicdomain/zero/1.0/ import hashlib import importlib.util import sys from pathlib import Path import pytest from mozunit import main SCRIPT = ( Path(__file__).resolve().parent.parent / "scripts" / "misc" / "android-gradle-dependencies" / "verify_dependencies.py" ) spec = importlib.util.spec_from_file_location("verify_dependencies", SCRIPT) verify_dependencies = importlib.util.module_from_spec(spec) spec.loader.exec_module(verify_dependencies) def digest(content): return hashlib.sha256(content).hexdigest() def write_inventory(directory, name, components): """Write an inventory in the shape `--write-verification-metadata` does. `components` maps "group:name:version" to {filename: content}, and the recorded checksum is that of the content, so a test makes a component corrupt or missing by writing something else, or nothing, to the repository. """ lines = [ '', '', " ", " true", " ", " ", ] for coordinates, files in components.items(): group, module, version = coordinates.split(":") lines.append( f' ' ) for filename, content in files.items(): lines.append(f' ') lines.append( f' ' ) lines.append(" ") lines.append(" ") lines += [" ", ""] directory.mkdir(parents=True, exist_ok=True) inventory = directory / f"{name}.xml" inventory.write_text("\n".join(lines), encoding="utf-8") return inventory def write_raw_inventory(directory, name, body): """An inventory written by hand, for shapes the helpers cannot express.""" directory.mkdir(parents=True, exist_ok=True) inventory = directory / f"{name}.xml" inventory.write_text( '' '' f"{body}", encoding="utf-8", ) return inventory def write_inventory_without_checksums(directory, name, components): """An inventory whose artifacts carry no , which verify() can only check for existence.""" body = "" for coordinates, files in components.items(): group, module, version = coordinates.split(":") artifacts = "".join(f'' for filename in files) body += ( f'' f"{artifacts}" ) return write_raw_inventory(directory, name, body) def populate(repository, components): """Lay components out in a repository the way a Maven repository does.""" for coordinates, files in components.items(): group, module, version = coordinates.split(":") directory = repository.joinpath(*group.split("."), module, version) directory.mkdir(parents=True, exist_ok=True) for filename, content in files.items(): (directory / filename).write_bytes(content) return repository def check(inventories, repositories, unproxied=()): inventory, _, _ = verify_dependencies.merge_inventories(sorted(inventories)) return verify_dependencies.verify(inventory, repositories, unproxied) def test_complete_tree(tmp_path): components = { "com.example:widget:1.0": { "widget-1.0.jar": b"jar", "widget-1.0.pom": b"pom", }, "androidx.core:core:1.2.3": {"core-1.2.3.aar": b"aar"}, } inventory = write_inventory(tmp_path / "inventories", "top-level", components) repository = populate(tmp_path / "central", components) result = check([inventory], [repository]) assert (result.problems, result.unpackaged) == ([], []) assert (result.artifacts, result.redirected) == (3, 0) def test_component_split_across_repositories(tmp_path): """Downstream tasks are handed every repository, so either will do.""" inventory = write_inventory( tmp_path / "inventories", "top-level", { "com.example:widget:1.0": {"widget-1.0.jar": b"jar"}, "androidx.core:core:1.2.3": {"core-1.2.3.aar": b"aar"}, }, ) central = populate( tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}} ) google = populate( tmp_path / "google", {"androidx.core:core:1.2.3": {"core-1.2.3.aar": b"aar"}} ) result = check([inventory], [central, google]) assert result.problems == [] assert (result.artifacts, result.redirected) == (2, 0) def test_inventories_are_merged(tmp_path): """Each pass covers one Gradle build; a shared component is packaged once.""" inventories = tmp_path / "inventories" shared = {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}} first = write_inventory( inventories, "top-level", {**shared, "com.example:only-top:1.0": {"only-top-1.0.jar": b"a"}}, ) second = write_inventory( inventories, "fenix", {**shared, "com.example:only-fenix:1.0": {"only-fenix-1.0.jar": b"b"}}, ) repository = populate( tmp_path / "central", { **shared, "com.example:only-top:1.0": {"only-top-1.0.jar": b"a"}, "com.example:only-fenix:1.0": {"only-fenix-1.0.jar": b"b"}, }, ) merged, counts, anomalies = verify_dependencies.merge_inventories([first, second]) assert len(merged) == 3 assert sorted(counts.values()) == [2, 2] assert anomalies == [] assert check([first, second], [repository]).problems == [] def test_file_published_by_another_module(tmp_path): """A component's metadata can name a file a different module publishes. Gradle records `guava-33.3.1-jre.jar` under `guava:33.3.1-android`, and Kotlin Multiplatform modules redirect to a platform-specific module. The bytes are packaged, just not where this component's coordinates point. """ inventory = write_inventory( tmp_path / "inventories", "top-level", { "com.google.guava:guava:33.3.1-android": { "guava-33.3.1-android.jar": b"android", "guava-33.3.1-jre.jar": b"jre", } }, ) repository = populate( tmp_path / "central", { "com.google.guava:guava:33.3.1-android": { "guava-33.3.1-android.jar": b"android" }, "com.google.guava:guava:33.3.1-jre": {"guava-33.3.1-jre.jar": b"jre"}, }, ) result = check([inventory], [repository]) assert result.problems == [] assert (result.artifacts, result.redirected) == (1, 1) def test_file_published_under_another_name(tmp_path): """The recorded name need not be the name on disk either, so fall back to matching the contents within the component's own directory.""" inventory = write_inventory( tmp_path / "inventories", "top-level", {"org.example.kmp:library:1.0": {"library-1.0.jar": b"bytes"}}, ) repository = populate( tmp_path / "central", {"org.example.kmp:library:1.0": {"library-jvm-1.0.jar": b"bytes"}}, ) result = check([inventory], [repository]) assert result.problems == [] assert (result.artifacts, result.redirected) == (0, 1) def test_missing_file(tmp_path): """The intermittent fault: the component is there, one of its files isn't.""" inventory = write_inventory( tmp_path / "inventories", "top-level", { "com.example:widget:1.0": { "widget-1.0.jar": b"jar", "widget-1.0.pom": b"pom", } }, ) repository = populate( tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.pom": b"pom"}} ) problems = check([inventory], [repository]).problems assert problems == [("missing", ("com.example", "widget", "1.0"), "widget-1.0.jar")] def test_corrupt_file(tmp_path): inventory = write_inventory( tmp_path / "inventories", "top-level", {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}}, ) repository = populate( tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"truncated"}}, ) problems = check([inventory], [repository]).problems assert len(problems) == 1 kind, component, detail = problems[0] assert (kind, component) == ("corrupt", ("com.example", "widget", "1.0")) assert detail.endswith("widget-1.0.jar") def test_absent_component(tmp_path): inventory = write_inventory( tmp_path / "inventories", "top-level", { "com.example:widget:1.0": {"widget-1.0.jar": b"jar"}, "com.example:absent:2.0": {"absent-2.0.jar": b"jar"}, }, ) repository = populate( tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}} ) problems = check([inventory], [repository]).problems # If this differs on some platform, fix the reporting, not the expectation. assert problems == [ ("absent", ("com.example", "absent", "2.0"), "com/example/absent/2.0") ] def test_unrecorded_files_are_ignored(tmp_path): """Nexus also stores what `mach build` resolved, which no pass enumerates.""" inventory = write_inventory( tmp_path / "inventories", "top-level", {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}}, ) repository = populate( tmp_path / "central", { "com.example:widget:1.0": { "widget-1.0.jar": b"jar", "widget-1.0.pom": b"pom", }, "com.example:unrecorded:1.0": {"unrecorded-1.0.jar": b"jar"}, }, ) assert check([inventory], [repository]).problems == [] @pytest.mark.parametrize( "inventory,expected", [ (None, "no inventories in"), ("", "list no components at all"), ], ids=["no-inventories", "no-components"], ) def test_main_rejects_an_unusable_inventory( tmp_path, monkeypatch, capsys, inventory, expected ): """A pass that did not run must not read as a cache with nothing missing.""" inventories = tmp_path / "inventories" inventories.mkdir() if inventory is not None: (inventories / "top-level.xml").write_text(inventory, encoding="utf-8") repository = tmp_path / "central" repository.mkdir() monkeypatch.setattr( sys, "argv", ["verify_dependencies.py", "--inventories", str(inventories), str(repository)], ) assert verify_dependencies.main() == 1 assert expected in capsys.readouterr().out def test_main_reports_a_complete_tree(tmp_path, monkeypatch, capsys): components = {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}} write_inventory(tmp_path / "inventories", "top-level", components) populate(tmp_path / "central", components) monkeypatch.setattr( sys, "argv", [ "verify_dependencies.py", "--inventories", str(tmp_path / "inventories"), str(tmp_path / "central"), ], ) assert verify_dependencies.main() == 0 out = capsys.readouterr().out assert "top-level: 1 components" in out assert "verified 1 artifacts across 1 of 1 components" in out def test_main_rejects_a_missing_repository(tmp_path, monkeypatch, capsys): """A repository that moved must not read as its components being absent.""" components = {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}} write_inventory(tmp_path / "inventories", "top-level", components) populate(tmp_path / "central", components) monkeypatch.setattr( sys, "argv", [ "verify_dependencies.py", "--inventories", str(tmp_path / "inventories"), str(tmp_path / "central"), str(tmp_path / "gone"), ], ) assert verify_dependencies.main() == 1 assert "not a packaged repository" in capsys.readouterr().out def test_unproxied_absence_is_expected(tmp_path): """GeckoView is built by this task, not cached, so it is absent by design.""" inventory = write_inventory( tmp_path / "inventories", "fenix", { "com.example:widget:1.0": {"widget-1.0.jar": b"jar"}, "org.mozilla.geckoview:geckoview-default-omni:1.2.3": { "geckoview-default-omni-1.2.3.aar": b"built here" }, }, ) repository = populate( tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"jar"}} ) result = check([inventory], [repository], ("org/mozilla/geckoview",)) assert result.problems == [] assert result.unpackaged == [ ("org.mozilla.geckoview", "geckoview-default-omni", "1.2.3") ] # Without the prefix it is a fault, and glean is not covered by it either. problems = check([inventory], [repository]).problems assert [kind for kind, _, _ in problems] == ["absent"] def test_unproxied_does_not_exempt_a_longer_named_sibling(tmp_path): """org/mozilla/geckoview must not cover org/mozilla/geckoviewextra.""" inventory = write_inventory( tmp_path / "inventories", "fenix", {"org.mozilla.geckoviewextra:widget:1.0": {"widget-1.0.jar": b"jar"}}, ) repository = tmp_path / "central" repository.mkdir() result = check([inventory], [repository], ("org/mozilla/geckoview",)) assert result.unpackaged == [] assert [kind for kind, _, _ in result.problems] == ["absent"] # The prefix itself, and a parent of it, do exempt. for prefix in ("org/mozilla/geckoviewextra", "org/mozilla"): result = check([inventory], [repository], (prefix,)) assert (result.problems, len(result.unpackaged)) == ([], 1), prefix def test_report_separates_the_kinds_and_their_advice(tmp_path, capsys): """Absent and incompletely packaged want different things from the reader.""" verify_dependencies.report([ ("absent", ("org.example", "gone", "1.0"), "org/example/gone/1.0"), ("missing", ("com.example", "widget", "1.0"), "widget-1.0.jar"), ("corrupt", ("com.example", "other", "2.0"), "/packaged/other-2.0.jar"), ]) out = capsys.readouterr().out # Treeherder's log parser surfaces the "FATAL ERROR" line and nothing else, # so the tally has to be on it. See treeherder/log_parser/parsers.py. fatal = [line for line in out.splitlines() if "FATAL ERROR" in line] assert fatal == [ "FATAL ERROR: the packaged dependency cache is incomplete: " "1 component absent, 1 file missing, 1 file corrupt." ] assert "1 component resolved by an enumeration pass" in out assert " absent: org.example:gone:1.0 (looked for org/example/gone/1.0)" in out assert "--unproxied" in out assert "1 file resolved from a component that is packaged" in out assert " missing: com.example:widget:1.0 -> widget-1.0.jar" in out assert "Re-running the task usually produces a complete one." in out assert "1 packaged file whose contents do not match" in out assert " corrupt: /packaged/other-2.0.jar" in out @pytest.mark.parametrize("prefix", ["", "."], ids=["empty", "dot"]) def test_main_rejects_an_empty_unproxied_prefix(tmp_path, monkeypatch, capsys, prefix): """An empty prefix is every component's parent, so it would excuse the tree.""" components = {"com.example:absent:1.0": {"absent-1.0.jar": b"jar"}} write_inventory(tmp_path / "inventories", "top-level", components) repository = tmp_path / "central" repository.mkdir() monkeypatch.setattr( sys, "argv", [ "verify_dependencies.py", "--inventories", str(tmp_path / "inventories"), "--unproxied", prefix, str(repository), ], ) assert verify_dependencies.main() == 1 out = capsys.readouterr().out assert "--unproxied needs a path prefix" in out # It must fail on the flag, not sail past it and exempt the missing component. assert "not packaged, as expected" not in out def test_conflicting_checksums_are_reported_but_not_fatal( tmp_path, monkeypatch, capsys ): """Only one file can be packaged and downstream resolves it happily, so the disagreement is worth saying without failing the task.""" inventories = tmp_path / "inventories" write_inventory( inventories, "top-level", {"com.example:widget:1.0": {"widget-1.0.jar": b"one"}} ) write_inventory( inventories, "fenix", {"com.example:widget:1.0": {"widget-1.0.jar": b"two"}} ) populate( tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"two"}} ) merged, _, anomalies = verify_dependencies.merge_inventories( sorted(inventories.glob("*.xml")) ) assert merged[("com.example", "widget", "1.0")]["widget-1.0.jar"] == { digest(b"one"), digest(b"two"), } assert anomalies == [ "com.example:widget:1.0 -> widget-1.0.jar is recorded with more than " "one checksum; any of them is accepted" ] monkeypatch.setattr( sys, "argv", [ "verify_dependencies.py", "--inventories", str(inventories), str(tmp_path / "central"), ], ) assert verify_dependencies.main() == 0 out = capsys.readouterr().out assert "WARNING: com.example:widget:1.0 -> widget-1.0.jar" in out assert "FATAL" not in out def test_a_pass_without_a_checksum_does_not_erase_one(tmp_path): """Otherwise the artifact stops being integrity-checked, and the redirect path reports it missing because it has nothing to match on.""" inventories = tmp_path / "inventories" component = { "com.google.guava:guava:33.3.1-android": {"guava-33.3.1-jre.jar": b"jre"} } # The checksumless pass must sort last, so that it is the one that would # overwrite: merge_inventories reads them in name order. write_inventory(inventories, "android-components", component) write_inventory_without_checksums(inventories, "top-level", component) # The -android component is packaged, but the file it records lives under # the module that publishes it, so only the checksum finds it. repository = populate( tmp_path / "central", { "com.google.guava:guava:33.3.1-android": { "guava-33.3.1-android.jar": b"android" }, "com.google.guava:guava:33.3.1-jre": {"guava-33.3.1-jre.jar": b"jre"}, }, ) merged, _, anomalies = verify_dependencies.merge_inventories( sorted(inventories.glob("*.xml")) ) assert anomalies == [] # The real checksum survives alongside the empty record, so the redirect # still has something to match on. assert merged[("com.google.guava", "guava", "33.3.1-android")] == { "guava-33.3.1-jre.jar": {None, digest(b"jre")} } result = check(sorted(inventories.glob("*.xml")), [repository]) assert result.problems == [] assert (result.artifacts, result.redirected) == (0, 1) def test_an_artifact_no_pass_checksummed_is_still_checked_for_existence(tmp_path): """Dropping it from the merge entirely would stop it being checked at all.""" inventories = tmp_path / "inventories" write_inventory_without_checksums( inventories, "top-level", {"com.example:widget:1.0": {"widget-1.0.jar": None}} ) repository = populate(tmp_path / "central", {"com.example:widget:1.0": {}}) merged, _, _ = verify_dependencies.merge_inventories( sorted(inventories.glob("*.xml")) ) assert merged == {("com.example", "widget", "1.0"): {"widget-1.0.jar": {None}}} problems = check(sorted(inventories.glob("*.xml")), [repository]).problems assert problems == [("missing", ("com.example", "widget", "1.0"), "widget-1.0.jar")] def test_one_artifact_recorded_twice_in_one_inventory(tmp_path): """Same disagreement as across passes, and the same non-fatal handling.""" inventory = write_raw_inventory( tmp_path / "inventories", "top-level", '' f'' f'' "", ) repository = populate( tmp_path / "central", {"com.example:widget:1.0": {"widget-1.0.jar": b"one"}} ) merged, _, anomalies = verify_dependencies.merge_inventories([inventory]) assert merged[("com.example", "widget", "1.0")]["widget-1.0.jar"] == { digest(b"one"), digest(b"two"), } assert len(anomalies) == 1 and "more than one checksum" in anomalies[0] # Either recorded file is accepted, so the packaged one is not "corrupt". assert check([inventory], [repository]).problems == [] @pytest.mark.parametrize( "body,expected", [ ( '' '', "a component has no coordinates", ), ( '' '', "records an artifact with no name", ), ], ids=["no-coordinates", "no-artifact-name"], ) def test_malformed_entries_are_reported_not_fatal(tmp_path, body, expected): """Skipping one entry checks less, but does not make the cache unusable.""" inventory = write_raw_inventory(tmp_path / "inventories", "top-level", body) merged, _, anomalies = verify_dependencies.merge_inventories([inventory]) assert len(anomalies) == 1 and expected in anomalies[0] # A skipped entry leaves nothing behind for verify() to trip over. assert verify_dependencies.verify(merged, [tmp_path]).problems == [] def test_an_exempt_component_stays_exempt_when_partly_packaged(tmp_path): """Whether this task packages a component is a property of its coordinates. A directory left behind at those coordinates must not turn the exemption into a fatal, but is worth saying in case the exemption is stale.""" inventory = write_inventory( tmp_path / "inventories", "fenix", { "org.mozilla.geckoview:geckoview-default-omni:1.2.3": { "geckoview-default-omni-1.2.3.aar": b"built here" } }, ) # The coordinates exist on disk, but without the file that was recorded. repository = populate( tmp_path / "central", {"org.mozilla.geckoview:geckoview-default-omni:1.2.3": {"leftover.txt": b"x"}}, ) result = check([inventory], [repository], ("org/mozilla/geckoview",)) assert result.problems == [] assert result.unpackaged == [ ("org.mozilla.geckoview", "geckoview-default-omni", "1.2.3") ] assert len(result.anomalies) == 1 assert "exempt from packaging but is partly packaged" in result.anomalies[0] def test_either_recorded_checksum_is_accepted(tmp_path): """The packaged file can match whichever record, not just the last one.""" inventories = tmp_path / "inventories" write_inventory( inventories, "android-components", {"com.example:widget:1.0": {"w-1.0.jar": b"one"}}, ) write_inventory( inventories, "top-level", {"com.example:widget:1.0": {"w-1.0.jar": b"two"}} ) # Matches the *first* record; the naive implementation kept only the last. repository = populate( tmp_path / "central", {"com.example:widget:1.0": {"w-1.0.jar": b"one"}} ) result = check(sorted(inventories.glob("*.xml")), [repository]) assert result.problems == [] assert result.artifacts == 1 def test_unproxied_takes_more_than_one_prefix(tmp_path): """The flag is repeatable, so more than one exemption has to work at once.""" inventory = write_inventory( tmp_path / "inventories", "fenix", { "org.mozilla.geckoview:gv:1.0": {"gv-1.0.aar": b"a"}, "com.example.local:built:2.0": {"built-2.0.jar": b"b"}, "com.example:packaged:3.0": {"packaged-3.0.jar": b"c"}, }, ) repository = populate( tmp_path / "central", {"com.example:packaged:3.0": {"packaged-3.0.jar": b"c"}} ) result = check( [inventory], [repository], ("org/mozilla/geckoview", "com/example/local") ) assert result.problems == [] assert sorted(result.unpackaged) == [ ("com.example.local", "built", "2.0"), ("org.mozilla.geckoview", "gv", "1.0"), ] # Only the named prefixes are exempt; a sibling under com/example is not. assert result.artifacts == 1 def test_only_component_elements_are_read(tmp_path): """Matched on the local name, so a nested element whose name merely ends in "component" is not mistaken for one.""" inventory = write_raw_inventory( tmp_path / "inventories", "top-level", '' f'' "" '' f'' "", ) components, anomalies = verify_dependencies.parse_inventory(inventory) assert list(components) == [("com.example", "real", "1.0")] assert anomalies == [] if __name__ == "__main__": main()