# Python: venvs, requirements lock files and the tooling managing them. Python is only ever run from # a venv created by the recipes of this file, see check_python_venv_usage. # # Python requirements lock files (*requirements.txt) are generated by uv from their *requirements.in # sources in universal mode: a lock file is valid on any platform and any Python version greater than # or equal to PYTHON_MIN_VERSION. Releases younger than PYTHON_RELEASE_COOLDOWN are not selected, # which leaves time for compromised releases to be reported and removed, versions already in a lock # file are kept though. To disregard the cooldown, e.g. to get a security fix immediately, run # 'make upgrade_python_requirements PYTHON_RELEASE_COOLDOWN='. uv and pip-audit are installed with # pip in a dedicated venv, their versions are pinned in # ci/python_tooling_requirements.in, any Python at least PYTHON_MIN_VERSION able to create a venv can # be used, set PYTHON_EXE to point to one if the default python3 is too old. PYTHON_MIN_VERSION=3.12 PYTHON_RELEASE_COOLDOWN=3 days PYTHON_EXE?=python3 # One venv per set of python requirements, created by a _setup_venv target defined next to its # users (see semgrep_lint_setup_venv or webdriver_setup_venv) with create_python_venv_with_requirements, # and PYTHON_VENV (default: venv) for python scripts needing no package, created by setup_python_venv: # that one never gets pip nor packages, so that scripts cannot silently depend on leftovers PY_TOOLING_VENV=venv_python_tooling # Arguments for a Python interpreter, exits with 0 if it is at least PYTHON_MIN_VERSION PYTHON_MIN_VERSION_CHECK=-c "import sys; sys.exit(0 if sys.version_info[:2] >= tuple(int(x) for x in '$(PYTHON_MIN_VERSION)'.split('.')) else 1)" # Shell snippet activating the venv at the given path, then Safe-Chain (see activate_safe_chain in # the main Makefile), in this order so that the Safe-Chain shims come first in the PATH and wrap # python and pip, called by name, for the rest of the recipe. define activate_python_venv . "$(1)/bin/activate" && \ $(activate_safe_chain) endef # Seconds pip waits for the network to answer (connection or data) before giving up on a request, # instead of its default of 15: packages are installed through Safe-Chain, which checks them before # handing them over to pip and can take longer than that to answer. PIP_DEFAULT_TIMEOUT is the # environment variable pip reads its --timeout option from: exporting it applies the value to every # pip call of every recipe, and it takes precedence over the pip configuration files, including the # one Safe-Chain generates. A value from the environment or the command line is used instead, e.g. # 'make ... PIP_DEFAULT_TIMEOUT=120'. export PIP_DEFAULT_TIMEOUT?=60 # Install the given *requirements.txt lock files with pip in the venv at the given path, refusing # anything that is not a venv, with the packages checked by Safe-Chain. The pinned pip is installed # first, the pip bundled with Python may be too old to install the lock files, see # ci/pip_requirements.in define pip_install_requirements @$(1)/bin/python -c 'import sys; sys.exit(0 if sys.prefix != sys.base_prefix else 1)' || \ ( echo "Refusing to install python packages in '$(1)', which is not a venv" && exit 1 ) $(call activate_python_venv,$(1)) && \ python -m pip install --quiet --disable-pip-version-check -r ci/pip_requirements.txt && \ python -m pip install $$(printf -- '-r %s ' $(2)) endef # Marker of the venvs created by create_python_venv, the only ones it may recreate PYTHON_VENV_MARKER=.created_by_makefile # Create a venv at the given path with PYTHON_EXE, which must be at least PYTHON_MIN_VERSION, without # pip so that nothing can be installed in it. An existing directory is refused unless it is a venv. # A venv created by this recipe is recreated if it was not created from PYTHON_EXE, e.g. because the # checkout is shared between a host and a container whose binaries are not interchangeable; a venv # created by something else is used as is when its interpreter is PYTHON_EXE and refused otherwise, # it is never cleared. define create_python_venv @$(PYTHON_EXE) $(PYTHON_MIN_VERSION_CHECK) 2> /dev/null || \ ( echo "Python $(PYTHON_MIN_VERSION) or newer is required to run python but \ '$(PYTHON_EXE)' is $$($(PYTHON_EXE) --version 2> /dev/null || echo "not found"). Install a newer Python and point \ PYTHON_EXE to it, e.g. 'make $@ PYTHON_EXE=python3.12' or 'export PYTHON_EXE=python3.12'." && exit 1 ) @if [ -e "$(1)" ] && [ ! -f "$(1)/pyvenv.cfg" ]; then \ echo "Refusing to use '$(1)' as a venv, it exists and is not one" && exit 1; \ fi @if [ -f "$(1)/pyvenv.cfg" ] && ! $(PYTHON_EXE) -c 'import os, sys; sys.exit(0 if os.path.realpath("$(1)/bin/python") == os.path.realpath(sys._base_executable) else 1)'; then \ if [ -f "$(1)/$(PYTHON_VENV_MARKER)" ]; then \ echo "$(1) was not created with $(PYTHON_EXE), recreating it"; \ $(PYTHON_EXE) -m venv --clear --without-pip "$(1)" && touch "$(1)/$(PYTHON_VENV_MARKER)"; \ else \ echo "Refusing to recreate '$(1)', a venv created neither with $(PYTHON_EXE) nor by this Makefile. Remove it or use another path." && exit 1; \ fi; \ fi @if [ -e "$(1)" ]; then \ $(PYTHON_EXE) -m venv --without-pip "$(1)"; \ else \ $(PYTHON_EXE) -m venv --without-pip "$(1)" && touch "$(1)/$(PYTHON_VENV_MARKER)"; \ fi endef # Create a venv at the given path (see create_python_venv), add pip to it and install the given # *requirements.txt lock files in it. Recipes are named differently from targets on purpose: # setup_python_venv is the target creating a bare venv. define create_python_venv_with_requirements $(call create_python_venv,$(1)) $(PYTHON_EXE) -m venv $(1) $(call pip_install_requirements,$(1),$(2)) endef .PHONY: python_min_version # Return the minimum Python version supported by the python tooling, installed by CI python_min_version: @echo "$(PYTHON_MIN_VERSION)" .PHONY: setup_python_venv # Create the venv PYTHON_VENV (default: venv) with PYTHON_EXE, without pip: nothing can be installed in it setup_python_venv: $(call create_python_venv,$(PYTHON_VENV)) .PHONY: check_python_venv_usage # Check that python is only run from a venv in the Makefiles and the workflows check_python_venv_usage: @./scripts/check_python_venv_usage.sh .PHONY: python_tooling_setup_venv # Create venv with pinned uv and pip-audit used to manage python requirements lock files python_tooling_setup_venv: $(call create_python_venv_with_requirements,$(PY_TOOLING_VENV),ci/python_tooling_requirements.txt) PYTHON_REQUIREMENTS_ENV=PY_UV_EXE="$(PY_TOOLING_VENV)/bin/uv" PY_PIP_AUDIT_EXE="$(PY_TOOLING_VENV)/bin/pip-audit" \ PYTHON_MIN_VERSION="$(PYTHON_MIN_VERSION)" PYTHON_RELEASE_COOLDOWN="$(PYTHON_RELEASE_COOLDOWN)" .PHONY: compile_python_requirements # Generate *requirements.txt lock files from *requirements.in, keeping existing pins compile_python_requirements: python_tooling_setup_venv $(PYTHON_REQUIREMENTS_ENV) ./scripts/python_requirements.sh compile .PHONY: upgrade_python_requirements # Generate *requirements.txt lock files from *requirements.in, upgrading all pins upgrade_python_requirements: python_tooling_setup_venv $(PYTHON_REQUIREMENTS_ENV) ./scripts/python_requirements.sh upgrade .PHONY: check_python_requirements # Check *requirements.txt lock files are up to date with their *requirements.in check_python_requirements: python_tooling_setup_venv $(PYTHON_REQUIREMENTS_ENV) ./scripts/python_requirements.sh check .PHONY: audit_python_requirements # Check *requirements.txt lock files for known vulnerabilities with pip-audit audit_python_requirements: python_tooling_setup_venv $(PYTHON_REQUIREMENTS_ENV) ./scripts/python_requirements.sh audit