import json import os import tempfile from tempfile import TemporaryDirectory import codegen import mozpack.path as mozpath from mozbuild.util import FileAvoidWrite AUTOGENERATED_HEADER = ( "/* THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. */\n\n" ) def intervention(filename, contents): if type(contents) is not str: [bug, origin] = filename.split("-") contents = { "label": "example.com", "bugs": {f"{bug}": {"matches": [f"*://{origin}/*"]}}, "interventions": [], } | contents for i in contents["interventions"]: if not "platforms" in i: i["platforms"] = ["all"] return { "path": f"data/interventions/{filename}.json", "contents": contents, } def injection(filename, contents): type = mozpath.splitext(filename)[1].lstrip(".") return { "path": f"injections/{type}/{filename}", "contents": contents, } def test( description, harness, inputs=[], expected_run_js=None, expected_generated_files={}, extra_generated_filenames=[], non_generated_files={}, ): with TemporaryDirectory(ignore_cleanup_errors=True) as base_addon_dir: interventions_dir = mozpath.join(base_addon_dir, "data", "interventions") generated_css_dir = mozpath.join( base_addon_dir, "injections", "generated", ) non_generated_css_dir = mozpath.join(base_addon_dir, "injections", "css") os.makedirs(interventions_dir) os.makedirs(generated_css_dir) os.makedirs(non_generated_css_dir) for filename, contents in non_generated_files.items(): full_path = mozpath.join(non_generated_css_dir, filename) with open(full_path, "w") as fd: fd.write(contents) for input in inputs: full_path = mozpath.join(base_addon_dir, input["path"]) contents = input["contents"] with open(full_path, "w") as fd: if type(contents) is not str: json.dump(input["contents"], fd) else: fd.write(input["contents"]) run_js_template_path = mozpath.join(base_addon_dir, "run.js") with open(run_js_template_path, "w") as run_js_template_fd: run_js_template_fd.write("AVAILABLE_INTERVENTIONS = {}") preprocessed_filenames = extra_generated_filenames + list( expected_generated_files.keys() ) with tempfile.NamedTemporaryFile(suffix="js") as final_runjs_fd: with FileAvoidWrite(final_runjs_fd.name) as output_run_js: codegen.generate_run_js( output_run_js, run_js_template_path, interventions_dir, *preprocessed_filenames, ) if expected_run_js is not None: run_js_contents = final_runjs_fd.read().decode("utf-8") try: run_js = run_js_contents.lstrip( "AVAILABLE_INTERVENTIONS = " ).strip() run_js = json.loads(run_js) harness.equals( expected_run_js, run_js, f"run.js was generated correctly for {description}", ) except json.decoder.JSONDecodeError: raise ValueError(f"Could not parse JSON from run.js:\n{run_js}") for filename, expected_contents in expected_generated_files.items(): generated_path = mozpath.join(generated_css_dir, filename) with FileAvoidWrite(generated_path) as generated_fd: codegen.generate_file(generated_fd, interventions_dir) with open(generated_path) as generated_fd: actual_contents = generated_fd.read() final_expected_contents = f"{AUTOGENERATED_HEADER}{expected_contents}" harness.equals( actual_contents, final_expected_contents, f"{filename} was generated with the correct contents for {description}", ) def WebCompatBuildTest(harness): intervention_without_generated_files = intervention( "100-test.com", { "interventions": [ { "content_scripts": {"css": ["bug100-test.css"]}, }, ] }, ) test( "no extra files generated if none are specified", harness, inputs=[ intervention_without_generated_files, injection("bug100-test.css", "body {}"), ], expected_run_js={ "100": intervention_without_generated_files["contents"], }, ) test( "expected css files are generated from css in json", harness, inputs=[ intervention( "100-example.com", { "css": { "test1": "c1 {}", "test2": "c2 {}", }, "interventions": [ {"css": ["test1"]}, { "css": { "all_frames": True, "match_origin_as_fallback": True, "which": ["test1", "test2"], } }, ], }, ), ], expected_run_js={ "100": { "label": "example.com", "bugs": { "100": {"matches": ["*://example.com/*"]}, }, "interventions": [ { "platforms": ["all"], "content_scripts": { "css": ["injections/generated/bug100-example.com-test1.css"] }, }, { "platforms": ["all"], "content_scripts": { "all_frames": True, "match_origin_as_fallback": True, "css": [ "injections/generated/bug100-example.com-test1.css", "injections/generated/bug100-example.com-test2.css", ], }, }, ], } }, expected_generated_files={ "bug100-example.com-test1.css": "c1 {}", "bug100-example.com-test2.css": "c2 {}", }, ) test( "filenames are cleaned up to prevent errors saving them on filesystem", harness, inputs=[ intervention( "100-example.com", { "css": { "(\u00e7 /\\testé)": "c1 {}", }, "interventions": [ {"css": ["(\u00e7 /\\testé)"]}, ], }, ), ], expected_run_js={ "100": { "label": "example.com", "bugs": { "100": {"matches": ["*://example.com/*"]}, }, "interventions": [ { "platforms": ["all"], "content_scripts": { "css": [ "injections/generated/bug100-example.com-c_teste.css" ] }, }, ], } }, expected_generated_files={ "bug100-example.com-c_teste.css": "c1 {}", }, ) harness.should_throw( "build aborts on invalid JSON files", "ValueError: 100-example.com.json is invalid JSON: Expecting value: line 1 column 1 (char 0)", lambda: test( "", harness, inputs=[intervention("100-example.com", "invalid JSON")], ), ) harness.should_throw( "build aborts on unused non-generated files", "ValueError: Please remove these files which are not referenced in any intervention JSON file: bug100-test.css", lambda: test( "", harness, non_generated_files={"bug100-test.css": ""}, ), ) harness.should_throw( "build aborts if unreferenced non-generated files are in-tree", "ValueError: Please remove these files which are not referenced in any intervention JSON file: bug100-test.json", lambda: test( "", harness, non_generated_files={"bug100-test.json": ""}, ), ) test( "allowed to have generic non-generated js/css files in-tree if they don't start with 'bug'", lambda: test( "", harness, non_generated_files={"generic-fix.css": ""}, ), ) harness.should_throw( "build aborts if preprocessed mozbuild is out of sync", "ValueError: preprocessed_intervention_files.mozbuild is out of date:\nPlease remove: bug100-example.old.com.css\nPlease add: bug100-example.com-test.css", lambda: test( "", harness, inputs=[ intervention( "100-example.com", { "css": { "test": "c {}", }, "interventions": [ {"css": ["test"]}, ], }, ), ], extra_generated_filenames=["bug100-example.old.com.css"], ), ) for invalid_css_section in [[], {}, 3, ""]: harness.should_throw( f"build aborts if an invalid value is given for a css section ({invalid_css_section})", "ValueError: css section should be a non-empty object or be removed from 100-example.com.json", lambda: test( "", harness, inputs=[ intervention( "100-example.com", { "css": invalid_css_section, }, ), ], ), ) harness.should_throw( "build aborts if no CSS fragments are given", "ValueError: css wanted, but none specified for 100-example.com.json", lambda: test( "", harness, inputs=[ intervention( "100-example.com", { "interventions": [ { "css": ["test1"], }, ], }, ), ], ), ) for css_section in [{"css": None}, {"css": {}}]: harness.should_throw( f"build aborts if bad CSS fragments are listed in intervention sections ({css_section})", "ValueError: css section should be a non-empty object or be removed from 100-example.com.json", lambda: test( "", harness, inputs=[ intervention( "100-example.com", { "interventions": [ { "css": ["test1"], }, ], } | css_section, ), ], ), ) for invalid_css_section in [[], {}, {"all_frames": True}]: harness.should_throw( "build aborts if intervention specifies invalid value for css section ({invalid_css_section})", "ValueError: intervention with missing `which` key or invalid array of desired css files in 100-example.com.json", lambda: test( "", harness, inputs=[ intervention( "100-example.com", { "css": {"test": "c {}"}, "interventions": [ {"css": ["test"]}, {"css": invalid_css_section}, ], }, ), ], ), ) for invalid_css_section in [[3], [True]]: harness.should_throw( f"build aborts if intervention specifies invalid value for css section ({invalid_css_section})", "ValueError: Empty or non-string filename not listed in intervention css section of 100-example.com.json", lambda: test( "", harness, inputs=[ intervention( "100-example.com", { "css": {"test": "c {}"}, "interventions": [ {"css": ["test"]}, {"css": invalid_css_section}, ], }, ), ], ), ) harness.should_throw( "build aborts if intervention specifies extra unknown keys in css section", "ValueError: unknown key(s) 'junk' in css section of 100-example.com.json", lambda: test( "", harness, inputs=[ intervention( "100-example.com", { "css": {"test": "c {}"}, "interventions": [ {"css": ["test"]}, {"css": {"which": ["test"], "junk": 3}}, ], }, ), ], ), ) for invalid_css_section in []: harness.should_throw( "build aborts if intervention specifies invalid value for css section ({invalid_css_section})", "ValueError: ", lambda: test( "", harness, inputs=[ intervention( "100-example.com", { "css": {"test": "c {}"}, "interventions": [ {"css": ["test"]}, {"css": invalid_css_section}, ], }, ), ], ), ) harness.should_throw( "build aborts if detects mixing of values for all_frames/match_origin_as_fallback", "ValueError: cannot mix value of all_frames in css and content_scripts sections in 100-example.com.json", lambda: test( "", harness, inputs=[ intervention( "100-example.com", { "css": { "test": "c {}", }, "interventions": [ { "content_scripts": { "all_frames": False, "match_origin_as_fallback": False, }, "css": { "all_frames": True, "match_origin_as_fallback": True, "which": ["test"], }, }, ], }, ), ], expected_generated_files={ "bug100-example.com-test.css": "c {}", }, ), ) harness.should_throw( "build aborts if cannot tell which JSON file to use", "ValueError: multiple json intervention files starting with 100.. not sure which to use from 100-example.com.json, 100-example2.com.json", lambda: test( "", harness, inputs=[ intervention( "100-example.com", { "css": { "test": "c {}", }, "interventions": [ {"css": ["test"]}, ], }, ), intervention( "100-example2.com", { "css": { "test": "c {}", }, "interventions": [ {"css": ["test"]}, ], }, ), ], expected_generated_files={ "bug100-example.com-test.css": "c {}", }, ), )