# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import argparse import atexit import os import re import shutil import sys from filter_git_changes import filter_git_changes from restore_patch_stack import restore_patch_stack from run_operations import ( ErrorHelp, RepoType, detect_repo_type, get_last_line, git_status, run_git, run_hg, run_shell, update_resume_state, ) from vendor_and_commit import vendor_and_commit # This script cherry-picks an upstream commit with the appropriate # commit message, and adds the no-op commit tracking file for the when # we vendor the upstream commit later. script_name = os.path.basename(__file__) error_help = ErrorHelp() error_help.set_prefix(f"*** ERROR *** {script_name} did not complete successfully") repo_type = detect_repo_type() def early_exit_handler(): error_help.print_help() def write_commit_message_file( commit_message_filename, github_path, github_sha, bug_number, reviewers, ): print(f"commit_message_filename: {commit_message_filename}") print(f"github_path: {github_path}") print(f"github_sha: {github_sha}") print(f"bug_number: {bug_number}") cmd = f"git show --format=%H --no-patch {github_sha}" stdout_lines = run_git(cmd, github_path) github_long_sha = stdout_lines[0] print(f"github_long_sha: {github_long_sha}") cmd = f"git show --format=%s%n%n%b --no-patch {github_sha}" github_commit_msg_lines = run_git(cmd, github_path) with open(commit_message_filename, "w") as ofile: ofile.write( f"Bug {bug_number} - Cherry-pick upstream libwebrtc commit {github_sha} r?{reviewers}" ) ofile.write("\n") ofile.write("\n") ofile.write( f"Upstream commit: https://webrtc.googlesource.com/src/+/{github_long_sha}" ) ofile.write("\n") for line in github_commit_msg_lines: ofile.write(f" {line}") ofile.write("\n") def cherry_pick_commit( commit_message_filename, github_path, github_sha, ): print(f"commit_message_filename: {commit_message_filename}") print(f"github_path: {github_path}") print(f"github_sha: {github_sha}") cmd = f"git cherry-pick --no-commit {github_sha}" run_git(cmd, github_path) cmd = f"git commit --file {os.path.abspath(commit_message_filename)}" run_git(cmd, github_path) def write_noop_tracking_file( github_sha, bug_number, ): noop_basename = f"{github_sha}.no-op-cherry-pick-msg" noop_filename = os.path.join(args.state_path, noop_basename) print(f"noop_filename: {noop_filename}") with open(noop_filename, "w") as ofile: ofile.write(f"We cherry-picked this in bug {bug_number}") ofile.write("\n") shutil.copy(noop_filename, args.patch_path) if repo_type == RepoType.GIT: cmd = f"git add {os.path.join(args.patch_path, noop_basename)}" run_git(cmd, ".") cmd = "git commit --amend --no-edit" run_git(cmd, ".") else: cmd = f"hg add {os.path.join(args.patch_path, noop_basename)}" run_hg(cmd) cmd = f"hg amend {os.path.join(args.patch_path, noop_basename)}" run_hg(cmd) if __name__ == "__main__": # first, check which repo we're in, git or hg if repo_type is None or not isinstance(repo_type, RepoType): print("Unable to detect repo (git or hg)") sys.exit(1) default_target_dir = "third_party/libwebrtc" default_state_dir = ".moz-fast-forward" default_log_dir = ".moz-fast-forward/logs" default_tmp_dir = ".moz-fast-forward/tmp" default_script_dir = "dom/media/webrtc/third_party_build" default_patch_dir = "third_party/libwebrtc/moz-patch-stack" default_repo_dir = ".moz-fast-forward/moz-libwebrtc" default_tar_name = "moz-libwebrtc.tar.gz" parser = argparse.ArgumentParser( description="Cherry-pick upstream libwebrtc commit" ) parser.add_argument( "--target-path", default=default_target_dir, help=f"target path for vendoring (defaults to {default_target_dir})", ) parser.add_argument( "--state-path", default=default_state_dir, help=f"path to state directory (defaults to {default_state_dir})", ) parser.add_argument( "--log-path", default=default_log_dir, help=f"path to log directory (defaults to {default_log_dir})", ) parser.add_argument( "--tmp-path", default=default_tmp_dir, help=f"path to tmp directory (defaults to {default_tmp_dir})", ) parser.add_argument( "--script-path", default=default_script_dir, help=f"path to script directory (defaults to {default_script_dir})", ) parser.add_argument( "--repo-path", default=default_repo_dir, help=f"path to moz-libwebrtc repo (defaults to {default_repo_dir})", ) parser.add_argument( "--tar-name", default=default_tar_name, help=f"name of tar file (defaults to {default_tar_name})", ) parser.add_argument( "--commit-sha", required=True, help="sha of commit to examine", ) parser.add_argument( "--branch", default="mozpatches", help="moz-libwebrtc branch (defaults to mozpatches)", ) parser.add_argument( "--commit-bug-number", type=int, required=True, help="integer Bugzilla number (example: 1800920)", ) parser.add_argument( "--patch-path", default=default_patch_dir, help=f"path to save patches (defaults to {default_patch_dir})", ) parser.add_argument( "--reviewers", required=True, help='reviewers for cherry-picked patch (like "ng,mjf")', ) parser.add_argument( "--abort", action="store_true", default=False, help="abort an interrupted cherry-pick", ) parser.add_argument( "--continue", dest="cont", # because args.continue causes syntax errors action="store_true", default=False, help="continue an interrupted cherry-pick", ) parser.add_argument( "--skip-restore", action="store_true", default=False, help="to skip restoring the patch-stack if it is already restored and verified", ) args = parser.parse_args() # register the exit handler after the arg parser completes so '--help' doesn't exit with # an error. atexit.register(early_exit_handler) commit_message_filename = os.path.join(args.tmp_path, "cherry-pick-commit_msg.txt") resume_state_filename = os.path.join(args.state_path, "cherry_pick_commit.resume") resume_state = "" if os.path.exists(resume_state_filename): resume_state = get_last_line(resume_state_filename).strip() print(f"resume_state: '{resume_state}'") # don't allow abort/continue flags if not in resume state error_help.set_help( "--abort or --continue flags are not allowed when not in resume state" ) if len(resume_state) == 0 and (args.abort or args.cont): sys.exit(1) error_help.set_help(None) # detect missing abort/continue flags if in resume state error_help.set_help("cherry-pick in progress, use --abort or --continue") if len(resume_state) != 0 and not args.abort and not args.cont: sys.exit(1) error_help.set_help(None) # handle aborting cherry-pick if args.abort: if repo_type == RepoType.GIT: run_git(f"git restore --staged {args.target_path}", ".") run_git(f"git restore {args.target_path}", ".") run_git(f"git clean -f {args.target_path}", ".") else: run_hg("hg revert --all") run_hg(f"hg purge {args.target_path}") # If the resume_state is not resume2 or resume3 that means we # may have committed something to the Mozilla repo. First we # need to check for our cherry-pick commit message, and if # found, remove that commit. if resume_state not in ("resume2", "resume3"): # check for committed patch and backout if repo_type == RepoType.GIT: stdout_lines = run_git("git show --oneline --no-patch", ".") else: stdout_lines = run_hg("hg log --template {desc|firstline}\n -r .") # check for "Cherry-pick upstream libwebrtc commit" print(f"stdout_lines before filter: {stdout_lines}") stdout_lines = [ line for line in stdout_lines if re.findall("Cherry-pick upstream libwebrtc commit", line) ] print(f"looking for commit: {stdout_lines}") if len(stdout_lines) > 0: if repo_type == RepoType.GIT: cmd = "git reset --hard HEAD^" print(f"calling '{cmd}'") run_git(cmd, ".") else: cmd = "hg prune ." print(f"calling '{cmd}'") run_hg(cmd) print("restoring patch stack") restore_patch_stack( args.repo_path, args.branch, os.path.abspath(args.patch_path), args.state_path, args.tar_name, ) # reset the resume file print("reset resume file") update_resume_state("", resume_state_filename) print("after resetting resume file") atexit.unregister(early_exit_handler) sys.exit(0) # make sure the relevant bits of the Mozilla repo are clean before # beginning error_help.set_help( f"There are modified or untracked files under {args.target_path}.\n" f"Please cleanup the repo under {args.target_path} before running {script_name}" ) if repo_type == RepoType.GIT: stdout_lines = git_status(".", args.target_path) else: stdout_lines = run_hg(f"hg status {args.target_path}") if len(stdout_lines) != 0: sys.exit(1) error_help.set_help(None) if len(resume_state) == 0: if args.skip_restore is False: # Restoring is done with each new cherry-pick to ensure that # guidance from verify_vendoring (the next step) is # accurate. If living dangerously is your thing, you can # skip this step. print("restoring patch stack") restore_patch_stack( args.repo_path, args.branch, os.path.abspath(args.patch_path), args.state_path, args.tar_name, ) update_resume_state("resume2", resume_state_filename) # make sure the github repo exists error_help.set_help( f"No moz-libwebrtc github repo found at {args.repo_path}\n" f"Please run restore_patch_stack.py before running {script_name}" ) if not os.path.exists(args.repo_path): sys.exit(1) error_help.set_help(None) # Other scripts assume the short-sha is used for various comparisons, so # make sure the provided sha is in the short form. cmd = f"git rev-parse --short {args.commit_sha}" args.commit_sha = run_git(cmd, args.repo_path)[0] if len(resume_state) == 0 or resume_state == "resume2": resume_state = "" update_resume_state("resume3", resume_state_filename) # Run verify_vendoring to make sure we have a sane patch-stack. print("verifying patch stack") run_shell(f"bash {args.script_path}/verify_vendoring.sh", False) if len(resume_state) == 0 or resume_state == "resume3": resume_state = "" update_resume_state("resume4", resume_state_filename) print("-------") print(f"------- write commit message file {commit_message_filename}") print("-------") write_commit_message_file( commit_message_filename, args.repo_path, args.commit_sha, args.commit_bug_number, args.reviewers, ) if len(resume_state) == 0 or resume_state == "resume4": resume_state = "" update_resume_state("resume5", resume_state_filename) print("-------") print(f"------- cherry-pick {args.commit_sha} into {args.repo_path}") print("-------") full_commit_message_filename = os.path.abspath(commit_message_filename) error_help.set_help( f"The cherry-pick operation of {args.commit_sha} has failed.\n" "To fix this issue, you will need to jump to the github\n" f"repo at {args.repo_path} .\n" "Please resolve all the cherry-pick conflicts, and commit the changes\n" "using:\n" f" git commit --file {full_commit_message_filename}\n" "\n" "When the github cherry-pick is complete, resume running this\n" f"script ({script_name})" ) cherry_pick_commit( commit_message_filename, args.repo_path, args.commit_sha, ) error_help.set_help(None) if len(resume_state) == 0 or resume_state == "resume5": resume_state = "" update_resume_state("resume6", resume_state_filename) print("-------") print(f"------- vendor from {args.repo_path}") print("-------") error_help.set_help( f"Vendoring the newly cherry-picked git commit ({args.commit_sha}) has failed.\n" "The Mozilla repo is in an unknown state. This failure is\n" "rare and thus makes it difficult to provide definitive guidance.\n" "In essence, the current failing command is:\n" f"./mach python {args.script_path}/vendor_and_commit.py \\\n" f" --script-path {args.script_path} \\\n" f" --repo-path {args.repo_path} \\\n" f" --branch {args.branch} \\\n" f" --commit-sha {args.commit_sha} \\\n" f" --target-path {args.target_path} \\\n" f" --state-path {args.state_path} \\\n" f" --log-path {args.log_path} \\\n" f" --commit-msg-path {commit_message_filename}\n" "\n" "Additional guidance may be in the terminal output above. Resolve\n" "issues encountered by vendor_and_commit.py followed by re-running\n" "vendor_and_commit.py to resume/complete its processing. After\n" "vendor_and_commit.py completes successfully, resume running\n" f"this script ({script_name})" ) vendor_and_commit( args.script_path, args.repo_path, args.branch, args.commit_sha, args.target_path, # os.path.abspath(args.target_path), args.state_path, args.log_path, commit_message_filename, ) error_help.set_help(None) if len(resume_state) == 0 or resume_state == "resume6": resume_state = "" update_resume_state("resume7", resume_state_filename) error_help.set_help( "Reverting change to 'third_party/libwebrtc/README.mozilla.last-vendor'\n" "has failed. The cherry-pick commit should not modify\n" "'third_party/libwebrtc/README.mozilla'. If necessary\n" "manually revert changes to 'third_party/libwebrtc/README.mozilla'\n" f"in the cherry-pick commit and re-run {script_name}\n" "to complete the cherry-pick processing." ) # The normal vendoring process updates README.mozilla with info # on what commit was vendored and the command line used to do # the vendoring. Since we're only reusing the vendoring script # here, we don't want to update the README.mozilla file. if repo_type == RepoType.GIT: cmd = ( "git checkout HEAD^ -- third_party/libwebrtc/README.mozilla.last-vendor" ) run_git(cmd, ".") cmd = "git commit --amend --no-edit" run_git(cmd, ".") else: cmd = "hg revert -r tip^ third_party/libwebrtc/README.mozilla.last-vendor" run_hg(cmd) cmd = "hg amend" run_hg(cmd) error_help.set_help(None) if len(resume_state) == 0 or resume_state == "resume7": resume_state = "" update_resume_state("resume8", resume_state_filename) # get the files changed from the newly vendored cherry-pick # commit in the Mozilla repo if repo_type == RepoType.GIT: cmd = "git show --format='' --name-status | grep -v 'README.'" else: cmd = "hg status --change tip --exclude '**/README.*'" stdout_lines = run_shell(cmd) # run_shell to allow file wildcard print(f"Mozilla repo changes:\n{stdout_lines}") mozilla_file_change_cnt = len(stdout_lines) # get the files changed from the original cherry-picked patch in # the libwebrtc github repo libwebrtc_paths_changed = filter_git_changes( args.repo_path, args.commit_sha, None ) print(f"Libwebrtc repo changes:\n{libwebrtc_paths_changed}") libwebrtc_file_change_cnt = len(libwebrtc_paths_changed) error_help.set_help( f"Vendoring the cherry-pick of commit {args.commit_sha} has failed due to mismatched\n" f"changed file counts between the Mozilla repo ({mozilla_file_change_cnt}) " f"and the libwebrtc repo ({libwebrtc_file_change_cnt}).\n" "This may be because the mozilla patch-stack was not verified after\n" "running restore_patch_stack.py. After reconciling the changes in\n" f"the newly committed patch, please re-run {script_name} to complete\n" "the cherry-pick processing." ) if mozilla_file_change_cnt != libwebrtc_file_change_cnt: sys.exit(1) error_help.set_help(None) if len(resume_state) == 0 or resume_state == "resume8": resume_state = "" update_resume_state("", resume_state_filename) print("-------") print("------- write the noop tracking file") print("-------") write_noop_tracking_file(args.commit_sha, args.commit_bug_number) # unregister the exit handler so the normal exit doesn't falsely # report as an error. atexit.unregister(early_exit_handler)