#!/usr/bin/env bash # Manage the *requirements.txt lock files generated by uv from their *requirements.in sources. # # Usage: python_requirements.sh # # compile Generate lock files, versions already present in a lock file are kept as long as they # satisfy the constraints of the .in file, which makes this operation idempotent # upgrade Generate lock files, upgrading all dependencies to their latest allowed versions # check Fail if a lock file is missing, has no .in source or differs from what compile produces # audit Fail if a lock file pins a version with a known vulnerability, using pip-audit # # Lock files are universal: they are valid on any platform and any Python version greater than or # equal to PYTHON_MIN_VERSION. Releases younger than PYTHON_RELEASE_COOLDOWN (e.g. "3 days") are not # selected, which leaves time for compromised releases to be reported and removed, except by compile # and check for packages already present in the lock file, whose versions were adopted on purpose. # Set PYTHON_RELEASE_COOLDOWN to an empty value to disregard the cooldown, e.g. to get a security fix # immediately. Both variables must be set. # PY_UV_EXE and PY_PIP_AUDIT_EXE can be set to the uv and pip-audit executables to use, they default # to the ones found in PATH. # When PY_AUDIT_REPORT_DIR is set, audit also writes a JSON report per lock file in that directory, # named after the lock file path with '/' replaced by '__', see ci/python_audit_report.py. set -euo pipefail MODE="${1:-}" if [[ "${MODE}" != "compile" && "${MODE}" != "upgrade" && "${MODE}" != "check" && "${MODE}" != "audit" ]]; then echo "Usage: $(basename "$0") " >&2 exit 2 fi : "${PYTHON_MIN_VERSION:?PYTHON_MIN_VERSION must be set to the minimum Python version supported by the lock files}" : "${PYTHON_RELEASE_COOLDOWN?PYTHON_RELEASE_COOLDOWN must be set to the minimum age of a release to be selected, or empty}" PY_UV_EXE="${PY_UV_EXE:-uv}" PY_PIP_AUDIT_EXE="${PY_PIP_AUDIT_EXE:-pip-audit}" # uv reads UV_* environment variables as settings, most of them change what gets resolved (indexes, # pre-releases, resolution strategy, cutoff date...) and would make the output depend on the # developer's setup, or let a package from another index into a lock file. Clear them, keeping only # the ones about connectivity and local storage. UV_ENV_KEPT=" UV_CACHE_DIR UV_NO_CACHE UV_HTTP_TIMEOUT UV_HTTP_RETRIES UV_NATIVE_TLS UV_INSECURE_HOST UV_OFFLINE UV_NO_PROGRESS UV_CONCURRENT_DOWNLOADS " for uv_variable in $(env | sed -n 's/^\(UV_[A-Za-z0-9_]*\)=.*/\1/p'); do if [[ "${UV_ENV_KEPT}" != *" ${uv_variable} "* ]]; then unset "${uv_variable}" fi done # Paths written in the generated files are relative to the current directory, always run from the # repository root so that the output does not depend on where the script is called from. # The assignment makes the script stop if git fails, 'cd "$(git ...)"' would silently not move. REPO_ROOT="$(git rev-parse --show-toplevel)" cd "${REPO_ROOT}" TMP_DIR="$(mktemp -d)" trap 'rm -rf "${TMP_DIR}"' EXIT # Write the list of tracked and untracked files matching a pattern to FILES_LIST, honoring .gitignore # so that requirements files from build directories or vendored dependencies are not picked up. The # list goes through a file instead of a pipe so that a git failure stops the script instead of # silently yielding an empty list. FILES_LIST="${TMP_DIR}/files" list_files() { git ls-files --cached --others --exclude-standard -z -- "$1" > "${FILES_LIST}" } uv_compile() { local requirements_in="$1" local requirements_txt="$2" shift 2 local cooldown_args=() if [[ -n "${PYTHON_RELEASE_COOLDOWN}" ]]; then cooldown_args+=(--exclude-newer "${PYTHON_RELEASE_COOLDOWN}") # Versions already in the lock file were adopted on purpose, possibly to get a security fix # before the cooldown elapsed: exempt their packages so that they are kept instead of being # downgraded. Not done when upgrading, which is precisely about selecting newer versions. if [[ "${MODE}" != "upgrade" && -f "${requirements_txt}" ]]; then local package while IFS= read -r package; do cooldown_args+=(--exclude-newer-package "${package}=2999-01-01") done < <(grep -oE '^[A-Za-z0-9_.-]+(\[[^]]*\])?==' "${requirements_txt}" | sed 's/\[.*//; s/==//' | sort -u) fi fi # --universal: single lock file for all platforms and Python versions >= PYTHON_MIN_VERSION, # using environment markers # --no-python-downloads: an interpreter is only needed to build source distributions, any local # one will do, no need to download one matching PYTHON_MIN_VERSION # --no-config: ignore uv configuration files (uv.toml, pyproject.toml), UV_* environment # variables are cleared above for the same reason # --default-index: resolve against PyPI only, the lock files pin the hashes of its files # --no-build: only use the metadata published on the index, never build a source distribution, # which would run code from the package during resolution # --exclude-newer: ignore releases younger than the cooldown, see above # --custom-compile-command: recorded in the header in place of the actual command line, so that # the output does not depend on how uv was invoked and documents how to regenerate the files # ${array[@]+"${array[@]}"} expands to nothing when the array is empty, "${array[@]}" would be an # unbound variable error with bash 3 (macOS) under 'set -u' "${PY_UV_EXE}" pip compile --universal --python-version "${PYTHON_MIN_VERSION}" \ --generate-hashes --no-python-downloads --no-config --no-build --quiet \ --default-index https://pypi.org/simple \ ${cooldown_args[@]+"${cooldown_args[@]}"} \ --custom-compile-command "make compile_python_requirements" "$@" \ --output-file "${requirements_txt}" "${requirements_in}" } STATUS=0 list_files '*requirements.txt' while IFS= read -r -d '' requirements_txt; do [[ -f "${requirements_txt}" ]] || continue requirements_in="${requirements_txt%.txt}.in" if [[ ! -f "${requirements_in}" ]]; then echo "ERROR: ${requirements_txt} has no ${requirements_in} source file." \ "Create it with the direct dependencies and run 'make compile_python_requirements'." >&2 STATUS=1 fi done < "${FILES_LIST}" list_files '*requirements.in' IN_FILES_COUNT=0 while IFS= read -r -d '' requirements_in; do [[ -f "${requirements_in}" ]] || continue IN_FILES_COUNT=$((IN_FILES_COUNT + 1)) requirements_txt="${requirements_in%.in}.txt" case "${MODE}" in compile) echo "Compiling ${requirements_txt}" uv_compile "${requirements_in}" "${requirements_txt}" ;; upgrade) echo "Upgrading ${requirements_txt}" uv_compile "${requirements_in}" "${requirements_txt}" --upgrade ;; check) if [[ ! -f "${requirements_txt}" ]]; then echo "ERROR: ${requirements_txt} is missing, run 'make compile_python_requirements'." >&2 STATUS=1 continue fi echo "Checking ${requirements_txt}" # Compile to a copy of the current lock file so that its pins are taken into account, # exactly like compile does, without touching the working tree. tmp_txt="$(mktemp "${TMP_DIR}/requirements.XXXXXX")" cp "${requirements_txt}" "${tmp_txt}" uv_compile "${requirements_in}" "${tmp_txt}" if ! diff -u -L "${requirements_txt}" -L "${requirements_txt} (expected)" \ "${requirements_txt}" "${tmp_txt}"; then echo "ERROR: ${requirements_txt} is not up to date, see diff above." >&2 STATUS=1 fi ;; audit) if [[ ! -f "${requirements_txt}" ]]; then echo "ERROR: ${requirements_txt} is missing, run 'make compile_python_requirements'." >&2 STATUS=1 continue fi echo "Auditing ${requirements_txt}" # The lock file is complete and hashed, there is nothing to resolve or install audit_args=(--require-hashes --disable-pip --progress-spinner off) if [[ -n "${PY_AUDIT_REPORT_DIR:-}" ]]; then mkdir -p "${PY_AUDIT_REPORT_DIR}" audit_args+=(--format json --output "${PY_AUDIT_REPORT_DIR}/${requirements_txt//\//__}.json") fi if ! "${PY_PIP_AUDIT_EXE}" "${audit_args[@]}" --requirement "${requirements_txt}"; then echo "ERROR: ${requirements_txt} pins a vulnerable version, see output above." >&2 STATUS=1 fi ;; esac done < "${FILES_LIST}" if [[ "${IN_FILES_COUNT}" == "0" ]]; then echo "ERROR: no *requirements.in file found in ${REPO_ROOT}, nothing was done." >&2 STATUS=1 fi if [[ "${STATUS}" != "0" && "${MODE}" == "check" ]]; then echo "Python requirements lock files are not up to date," \ "run 'make compile_python_requirements' and commit the result." >&2 elif [[ "${STATUS}" != "0" && "${MODE}" == "audit" ]]; then echo "Python requirements lock files pin vulnerable versions, run" \ "'make upgrade_python_requirements' or bump the pinned versions in the .in files." >&2 fi exit "${STATUS}"