# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """ DevTools presubmit script See http://goo.gle/devtools-testing-guide#Presubmit-checks for more how to run presubmit checks in DevTools. See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for more details about the presubmit API built into gcl. `git cl presubmit -v -v` to debug presubmit checks. """ import re import sys import six import time from pathlib import Path USE_PYTHON3 = True PRESUBMIT_VERSION = '2.0.0' _EXCLUDED_PATHS = [ r'^extensions[\\/]cxx_debugging[\\/]e2e[\\/]runner\.py$', # Lines too long r'^front_end[\\/]core[\\/]common[\\/]Color\.ts$', # Apple copyright r'^front_end[\\/]core[\\/]common[\\/]Object\.ts$', # Apple copyright r'^front_end[\\/]core[\\/]common[\\/]ResourceType\.ts$', # Apple copyright # Apple copyright r'^front_end[\\/]core[\\/]dom_extension[\\/]DOMExtension\.ts$', r'^front_end[\\/]core[\\/]platform[\\/]UIString\.ts$', # Apple copyright r'^front_end[\\/]core[\\/]sdk[\\/]Resource\.ts$', # Apple copyright r'^front_end[\\/]core[\\/]sdk[\\/]Script\.ts$', # Apple copyright r'^front_end[\\/]third_party[\\/].*', # 3rd party code # Apple copyright r'^front_end[\\/]ui[\\/]legacy[\\/]components[\\/]data_grid[\\/]DataGrid\.ts$', r'^node_modules[\\/].*', # 3rd party code r'^scripts[\\/]build[\\/]build_inspector_overlay\.py$', # Lines too long r'^scripts[\\/]build[\\/]code_generator_frontend\.py$', r'^scripts[\\/]deps[\\/]manage_node_deps\.py$', # Lines too long r'front_end[\\/]generated[\\/]ARIAProperties\.ts$' # Auto-generated files # Auto-generated files r'front_end[\\/]generated[\\/]InspectorBackendCommands\.ts$' ] def _ExecuteSubProcess(input_api, output_api, script_path, script_arguments=None, message=None): if isinstance(script_path, six.string_types): script_path = [input_api.python3_executable, script_path] start_time = time.time() process = input_api.subprocess.Popen(script_path + (script_arguments or []), stdout=input_api.subprocess.PIPE, stderr=input_api.subprocess.STDOUT) out, _ = process.communicate() end_time = time.time() time_difference = end_time - start_time return [ output_api.PresubmitError( "%s (%.1fs): %s" % (message if message is not None else script_path, time_difference, out.decode('utf-8').strip())) ] if process.returncode != 0 else [] def _IsEnvCog(input_api): old_sys_path = sys.path[:] devtools_root = input_api.PresubmitLocalPath() depot_tools = input_api.os_path.join(devtools_root, 'third_party', 'depot_tools') try: sys.path.append(depot_tools) from gclient_utils import IsEnvCog if IsEnvCog(): return True finally: sys.path = old_sys_path return False def _GetAffectedFiles(input_api, parent_directories, excluded_actions, accepted_endings): """Return absolute file paths of affected files (not due to an excluded action) under a parent directory with an accepted file ending. """ local_paths = [ f.AbsoluteLocalPath() for f in input_api.AffectedFiles() if all(f.Action() != action for action in excluded_actions) ] affected_files = [ file_name for file_name in local_paths if any(parent_directory in file_name for parent_directory in parent_directories) and ( len(accepted_endings) == 0 or any( file_name.endswith(accepted_ending) for accepted_ending in accepted_endings)) ] return affected_files def _CheckWithNodeScript(input_api, output_api, script_path, script_arguments=None, allow_typescript=False, message=None): original_sys_path = sys.path try: sys.path = sys.path + [ input_api.os_path.join(input_api.PresubmitLocalPath(), 'scripts') ] import devtools_paths finally: sys.path = original_sys_path process = [devtools_paths.node_path(), script_path] if allow_typescript: process.insert(1, '--no-warnings=ExperimentalWarning') process.insert(1, '--experimental-strip-types') return _ExecuteSubProcess(input_api, output_api, process, script_arguments=script_arguments, message=message) def _GetFilesToLint(input_api, lint_config_files, default_linted_directories, accepted_endings): run_full_check = False files_to_lint = [] # We are changing the lint configuration; run the full check. if len(lint_config_files) != 0: run_full_check = True else: # Only run the linter on files that are relevant, to save PRESUBMIT time. files_to_lint = _GetAffectedFiles(input_api, default_linted_directories, ['D'], accepted_endings) # Exclude front_end/third_party and front_end/generated files. files_to_lint = [ file for file in files_to_lint if "front_end/third_party" not in file and "front_end/generated" not in file ] should_bail_out = len(files_to_lint) == 0 and not run_full_check return should_bail_out, files_to_lint def _CheckFormat(input_api, output_api): if _IsEnvCog(input_api): return [ output_api.PresubmitPromptWarning( 'Non-git environment detected, skipping _CheckFormat.') ] return _ExecuteSubProcess(input_api, output_api, ['git', 'cl', 'format', '--js'], message='Format') def CheckBugAssociationOnCommit(input_api, output_api): results = [] bugs = input_api.change.BugsFromDescription() message = ( "Each CL should be associated with a bug, use \'Bug:\' or \'Fixed:\' lines in\n" "the footer of the commit description. If you explicitly don\'t want to\n" "set a bug, use \'Bug: none\' in the footer of the commit description.\n\n" "Note: The footer of the commit description is the last block of lines in\n" "the commit description that doesn't contain empty lines. This means that\n" "any \'Bug:\' or \'Fixed:\' lines that are eventually followed by an empty\n" "line are not detected by this presubmit check.") if not bugs: results.append(output_api.PresubmitError(message)) return results def CheckExperimentTelemetry(input_api, output_api): experiment_telemetry_files = [ input_api.os_path.join(input_api.PresubmitLocalPath(), 'front_end', 'entrypoints', 'main', 'MainImpl.ts'), input_api.os_path.join(input_api.PresubmitLocalPath(), 'front_end', 'core', 'host', 'UserMetrics.ts') ] affected_main_files = _GetAffectedFiles(input_api, experiment_telemetry_files, [], ['.ts']) if len(affected_main_files) == 0: return [] script_path = input_api.os_path.join(input_api.PresubmitLocalPath(), 'scripts', 'check_experiments.js') return _CheckWithNodeScript(input_api, output_api, script_path, message='Experiment telemetry') def CheckESBuildVersion(input_api, output_api): script_path = input_api.os_path.join(input_api.PresubmitLocalPath(), 'scripts', 'check_esbuild_versions.js') return _CheckWithNodeScript(input_api, output_api, script_path, message='ESBuild version') def CheckDevToolsLint(input_api, output_api): lint_path = input_api.os_path.join(input_api.PresubmitLocalPath(), 'scripts', 'test', 'run_lint_check.mjs') front_end_directory = input_api.os_path.join( input_api.PresubmitLocalPath(), 'front_end') inspector_overlay_directory = input_api.os_path.join( input_api.PresubmitLocalPath(), 'inspector_overlay') test_directory = input_api.os_path.join(input_api.PresubmitLocalPath(), 'test') scripts_directory = input_api.os_path.join(input_api.PresubmitLocalPath(), 'scripts') default_linted_directories = [ front_end_directory, test_directory, scripts_directory, inspector_overlay_directory, ] lint_related_files = [ input_api.os_path.join(input_api.PresubmitLocalPath(), 'eslint.config.mjs'), input_api.os_path.join(input_api.PresubmitLocalPath(), '.stylelintrc.json'), input_api.os_path.join(input_api.PresubmitLocalPath(), '.stylelintignore'), # This file includes the LitAnalyzer rules input_api.os_path.join(input_api.PresubmitLocalPath(), 'tsconfig.json'), input_api.os_path.join(scripts_directory, 'test', 'run_lint_check.mjs'), ] lint_related_directories = [ input_api.os_path.join(input_api.PresubmitLocalPath(), 'node_modules'), ] lint_config_files = _GetAffectedFiles( input_api, lint_related_directories, [], [".js", ".mjs", ".ts"]) + _GetAffectedFiles( input_api, lint_related_files, [], []) should_bail_out, files_to_lint = _GetFilesToLint( input_api, lint_config_files, default_linted_directories, ['.css', '.mjs', '.js', '.ts']) if should_bail_out: return [] # If there are more than 50 files to check, don't bother and check # everything, so as to not run into command line length limits on Windows. if len(files_to_lint) > 50: files_to_lint = [] results = [] results.extend( _CheckWithNodeScript(input_api, output_api, lint_path, script_arguments=files_to_lint, allow_typescript=True, message="Lint")) results.extend(_CheckFormat(input_api, output_api)) return results def CheckGeneratedFiles(input_api, output_api): v8_directory_path = input_api.os_path.join(input_api.PresubmitLocalPath(), 'v8') blink_directory_path = input_api.os_path.join( input_api.PresubmitLocalPath(), 'third_party', 'blink') scripts_build_path = input_api.os_path.join(input_api.PresubmitLocalPath(), 'scripts', 'build') scripts_generated_output_path = input_api.os_path.join( input_api.PresubmitLocalPath(), 'front_end', 'generated') generated_aria_path = input_api.os_path.join(scripts_build_path, 'generate_aria.py') generated_supported_css_path = input_api.os_path.join( scripts_build_path, 'generate_supported_css.py') generated_deprecation_path = input_api.os_path.join( scripts_build_path, 'generate_deprecations.py') generated_protocol_path = input_api.os_path.join( scripts_build_path, 'code_generator_frontend.py') generated_protocol_typescript_path = input_api.os_path.join( input_api.PresubmitLocalPath(), 'scripts', 'protocol_typescript') concatenate_protocols_path = input_api.os_path.join( input_api.PresubmitLocalPath(), 'third_party', 'inspector_protocol', 'concatenate_protocols.py') affected_files = _GetAffectedFiles(input_api, [ v8_directory_path, blink_directory_path, input_api.os_path.join(input_api.PresubmitLocalPath(), 'third_party', 'pyjson5'), generated_aria_path, generated_supported_css_path, generated_deprecation_path, concatenate_protocols_path, generated_protocol_path, scripts_generated_output_path, generated_protocol_typescript_path, ], [], ['.pdl', '.json5', '.py', '.js', '.ts']) if len(affected_files) == 0: return [] generate_protocol_resources_path = input_api.os_path.join( input_api.PresubmitLocalPath(), 'scripts', 'deps', 'generate_protocol_resources.py') return _ExecuteSubProcess(input_api, output_api, generate_protocol_resources_path, message='Generated files') def CheckL10nStrings(input_api, output_api): devtools_root = input_api.PresubmitLocalPath() devtools_front_end = input_api.os_path.join(devtools_root, 'front_end') script_path = input_api.os_path.join(devtools_root, 'third_party', 'i18n', 'check-strings.js') affected_front_end_files = _GetAffectedFiles( input_api, [devtools_front_end, script_path], [], ['.js', '.ts']) if len(affected_front_end_files) == 0: return [] return _CheckWithNodeScript(input_api, output_api, script_path, [devtools_front_end], message='l10n strings') def CheckForTooLargeFiles(input_api, output_api): """Avoid large files, especially binary files, in the repository since git doesn't scale well for those. They will be in everyone's repo clones forever, forever making Chromium slower to clone and work with.""" # Uploading files to cloud storage is not trivial so we don't want # to set the limit too low, but the upper limit for "normal" large # files seems to be 1-2 MB, with a handful around 5-8 MB, so # anything over 20 MB is exceptional. TOO_LARGE_FILE_SIZE_LIMIT = 20 * 1024 * 1024 # 10 MB too_large_files = [] for f in input_api.AffectedFiles(): # Check both added and modified files (but not deleted files). if f.Action() in ('A', 'M'): size = input_api.os_path.getsize(f.AbsoluteLocalPath()) if size > TOO_LARGE_FILE_SIZE_LIMIT: too_large_files.append("%s: %d bytes" % (f.LocalPath(), size)) if too_large_files: message = ( 'Do not commit large files to git since git scales badly for those.\n' + 'Instead put the large files in cloud storage and use DEPS to\n' + 'fetch them.\n' + '\n'.join(too_large_files)) return [ output_api.PresubmitError('Too large files found in commit', long_text=message + '\n') ] return [] def CheckObsoleteScreenshotGoldens(input_api, output_api): script_path = input_api.os_path.join(input_api.PresubmitLocalPath(), 'scripts', 'test', 'check_obsolete_goldens.js') return _CheckWithNodeScript(input_api, output_api, script_path, script_arguments=[], message='Obsolete screenshot images') def CheckNodeModules(input_api, output_api): files = ['.clang-format', 'OWNERS', 'README.chromium'] results = [] for file in files: file_path = input_api.os_path.join(input_api.PresubmitLocalPath(), 'node_modules', file) if not Path(file_path).is_file(): results.extend([ output_api.PresubmitError( "node_modules/%s is missing. Use `npm run install-deps` to re-create it." % file) ]) node_module_files = _GetAffectedFiles(input_api, [ input_api.os_path.join(input_api.PresubmitLocalPath(), 'node_modules') ], [], []) # If the changes are above 100 assume that touching the node_modules # was intentional if len(node_module_files) == 0 or len(node_module_files) > 100: return results message = ( "Changes to `node_modules` detected.\n" + "This is third party code and should not be modified.\n" + "`node_module` are mainly used in testing infra\n" + "For bug fixes and features usually you should not need this change.\n" + "Was this change intentional?") results.extend([ output_api.PresubmitPromptWarning( message, locations=[ output_api.PresubmitResultLocation( # Location expects relative path # But _GetAffectedFiles returns us absolute path input_api.os_path.relpath( node_module_files[0], input_api.PresubmitLocalPath()), ), ]) ]) return results def CheckNoUncheckedFiles(input_api, output_api): if _IsEnvCog(input_api): return [] process = input_api.subprocess.Popen(['git', 'diff', '--exit-code'], stdout=input_api.subprocess.PIPE, stderr=input_api.subprocess.STDOUT) out, _ = process.communicate() if process.returncode != 0: files_changed_process = input_api.subprocess.Popen( ['git', 'diff'], stdout=input_api.subprocess.PIPE, stderr=input_api.subprocess.STDOUT) files_changed, _ = files_changed_process.communicate() return [ output_api.PresubmitError( 'You have changed files that need to be committed:\n%s' % (files_changed.decode('utf-8').strip())) ] return [] def CheckKnownContextValues(input_api, output_api): """Ensure all additions to `KnownContextValues.ts` following the naming convention. This check ensures that all new cases added to the enum in `KnownContextValues.ts` follow the extended Kebab Case naming convention. Specifically it doesn't look at unchanged lines, because there are various existing values that cannot be changed (easily). """ # This regexp matches the one we use in `StringUtilities.isExtendedKebabCase()`. kebab_case_re = re.compile( r"^([a-z0-9]+(?:-[a-z0-9]+)*\.)*[a-z0-9]+(?:-[a-z0-9]+)*$") local_path = input_api.os_path.join('front_end', 'ui', 'visual_logging', 'KnownContextValues.ts') invalid_contexts = [] for f in filter( lambda x: (x.LocalPath() == local_path and x.Action() == 'M'), input_api.AffectedFiles()): # Loop only through the changed lines of the affected file. for _, line in f.ChangedContents(): match = re.search(r"\s+'(.+)',", line) if match: context = match.group(1) if not kebab_case_re.match(context): invalid_contexts.append(context) continue if not invalid_contexts: return [] return [ output_api.PresubmitError( message=f"Invalid jslog context(s): {', '.join(invalid_contexts)}", long_text= ("""The jslog contexts must follow the extended Kebab Case naming convention, where words are separated with either a dash (`-`) or a dot (`.`), and all characters must be lower-case alphanumeric. """), locations=[ output_api.PresubmitResultLocation(file_path=local_path) ], ) ] # Canned check wrappers below. def _TextFilesOnlyFilter(file): """Filter that yields only text files. Filters files based on prefixes and extensions that should not be treated as text files for the canned checks below. """ excluded_prefixes = [ 'node_modules', 'third_party', 'front_end/third_party', 'extensions/cxx_debugging/third_party', ] excluded_extensions = [ '.png', '.webm', '.svg', '.avif', '.rawresponse', '.gz' ] if any(file.LocalPath().startswith(prefix) for prefix in excluded_prefixes): return False if any(file.LocalPath().endswith(ext) for ext in excluded_extensions): return False return True def CheckChangeHasNoCrAndHasOnlyOneEol(input_api, output_api): return input_api.canned_checks.CheckChangeHasNoCrAndHasOnlyOneEol( input_api, output_api, source_file_filter=_TextFilesOnlyFilter) def CheckGenderNeutral(input_api, output_api): return input_api.canned_checks.CheckGenderNeutral(input_api, output_api) def CheckAuthorizedAuthor(input_api, output_api): return input_api.canned_checks.CheckAuthorizedAuthor( input_api, output_api, bot_allowlist=[ 'chrome-cherry-picker@chops-service-accounts.iam.gserviceaccount.com', 'devtools-ci-autoroll-builder@chops-service-accounts.iam.gserviceaccount.com', ]) def CheckPanProjectChecksOnCommit(input_api, output_api): return input_api.canned_checks.PanProjectChecks( input_api, output_api, excluded_paths=_EXCLUDED_PATHS, maxlen=120)