#!/usr/bin/no-execute # shellcheck shell=bash # SPDX-License-Identifier: EPL-1.0 ############################################################################## # Copyright (c) 2019 The Linux Foundation and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Eclipse Public License v1.0 # which accompanies this distribution, and is available at # http://www.eclipse.org/legal/epl-v10.html ############################################################################## # # A library of functions for LF/Jenkins bash scripts. In the general case, these # functions should only use 'local' variables, and should NOT set # shell/environment variables. If you want to make a variable available, provide # a function that sets the variable: 'function lf_set_foo() {foo=asdf;}'. Any # scripts that need access to the variable can call the 'set' function. This # keeps the name-space pollution to a minimum. # # This script will be installed in ~jenkins by the Jenkins Init Script when the # build agent boots. When the build starts it should already be installed. # ################################################################################ # # NAME # lf-echo-stderr # # SYNOPSIS # source ~/lf-env.sh # # lf-echo-stderr "this entire" "string will be sent to stderr" # # DESCRIPTION # This function will echo all command line aruments to 'stderr' # # RETURN VALUE # None # ################################################################################ lf-echo-stderr () { echo "$@" 1>&2 } ################################################################################ # # NAME # lf-venv-pip-install # # SYNOPSIS # source ~/lf-env.sh # # lf-venv-pip-install "$lf_venv/bin/python3" $pip_opts virtualenv # # DESCRIPTION # Install packages with 'pip' into the venv belonging to the python # interpreter passed as the first argument. All remaining arguments are # handed to 'pip install' unmodified. # # Build agents set an 'index-url' in ~/.config/pip/pip.conf that points at # an internal PyPI mirror. While that mirror is unreachable every package # resolves to "No matching distribution found", and retrying against the # same index cannot succeed. This function therefore retries once against # upstream PyPI before giving up. # # RETURN VALUE # OK: 0 # Fail: 1 # ################################################################################ lf-venv-pip-install () { local venv_python="$1"; shift local upstream="https://pypi.org/simple" if "$venv_python" -m pip install "$@"; then return 0 fi # The only fallback worth attempting is a different index: a mirror that # is down fails identically no matter how many times it is retried. lf-echo-stderr "${FUNCNAME[0]}(): WARNING: install failed from the" \ "configured index, retrying against $upstream" "$venv_python" -m pip install --index-url "$upstream" "$@" } ################################################################################ # # NAME # lf-boolean # # SYNOPSIS # # shellcheck disable=SC1090 # source ~/lf-env.sh # # if lf-boolean $VAR; then # echo "VAR is true" # fi # # DESCRIPTION # This function will return a BOOLEAN (true or false) based upon the value # of VAR. The value of VAR will be mapped to lower case. If VAR maps to # "true", return true(0). If VAR maps to "false", return false(1). Any # other values will return false(2) and an error message. # # RETURN VALUES # OK: 0 # Fail: 1 or 2 # ################################################################################ lf-boolean () { if (( $# != 1 )); then echo "ERROR: ${FUNCNAME[0]}() line: ${BASH_LINENO[0]} :"\ " Missing Required Argument" return 1 fi local bool bool=$(echo "$1" | tr '[:upper:]' '[:lower:]') case $bool in true) return 0 ;; false) return 1 ;; '') lf-echo-stderr "ERROR: ${FUNCNAME[0]}() line:{BASH_LINENO[0]} :"\ " A boolean cannot be a empty string" >&2 return 2 ;; *) lf-echo-stderr "ERROR: ${FUNCNAME[0]}() line: ${BASH_LINENO[0]} :"\ " Invalid value for a boolean: '$bool'" return 2 ;; esac } ################################################################################ # # NAME # lf-activate-venv [-p|--python python] [-v|--venv-file] [--no-path] # [--system-site-packages] [package]... # # SYNOPSIS # # shellcheck disable=SC1090 # source ~/lf-env.sh # # lf-activate-venv tox tox-pyenv # or # lf-activate-venv jenkins-job-builder # or # lf-activate-venv lftools # or # lf-activate-venv --python python3.8 git-review # # lf-activate-venv --python python3.8 --venv-file /tmp/.myvenv git-review # # DESCRIPTION # This function will create a new Python Virtual Environment (venv) and # install the specified packages in the new venv. The venv will be installed # in $lf_venv and by default, the $lf_venv/bin directory will be prepended # to the PATH. # # The 'lf_venv' variable will be set so you can directly execute commands # in the venv with: $lf_venv/bin/command. lf-activate-venv() will check for # existing file '/tmp/.os_lf_venv' and set 'lf_venv' if the file exists. # # The function provides a --venv-file path for saving the value of the 'lf_env' # that can re-used later. By default '/tmp/.os_lf_venv' venv file is created # when the --venv-file option is not specified. # # Subsequent calls to lf-activate-venv() will re-use the existing venv # throught and will NOT overwrite 'lf_venv', if the '/tmp/.os_lf_venv' # already exists. # # If a new venv is required delete the file '/tmp/.os_lf_venv' before # calling lf-activate-venv() will create a fresh venv. # # By default all packages are installed with '--upgrade-strategy eager'. # The venv will always contain pip & virtualenv. # # Some packages have a default version. If one of those packages is specified, # the 'version' specifier will be added for the install. If the version is # specified on the command line that version will be used. # The following packages have default versions: # Package Version # jenkins-job-builder $JJB_VERSION # # If the --python flag is specified, the specified python executable will be # used to create the venv. The --python option must be in the PATH. The venv # will be located in $lf_venv (/tmp/venv-####). # # If the --no-path flag is specified, $lf_venv/bin will not be prepended to # the PATH. # # If the --system-site-packages flag is specified, the --system-site-packages # flag will be passed to the inital 'pip install' (python3* only). # # RETURN VALUES # OK: 0 # Fail: 1 # ################################################################################ lf-activate-venv () { lf_venv=$(mktemp -d /tmp/venv-XXXX) local venv_file="/tmp/.os_lf_venv" local python=python3 local options local set_path=true local install_args="" # set -x options=$(getopt -o 'np:v:' -l 'no-path,system-site-packages,python:,venv-file:' \ -n "${FUNCNAME[0]}" -- "$@" ) eval set -- "$options" while true; do case $1 in -n|--no-path) set_path=false ; shift ;; -p|--python) python="$2" ; shift 2 ;; -v|--venv-file) venv_file="$2" ; shift 2 ;; --system-site-packages) install_args="--system-site-packages" ; shift ;; --) shift; break ;; *) lf-echo-stderr \ "${FUNCNAME[0]}(): ERROR: Unknown switch '$1'." ; return 1 ;; esac done case $python in python2*) local pkg_list="$*" # For Python2, just create venv and install pip virtualenv -p "$python" "$lf_venv" || return 1 "$lf_venv/bin/pip" install --upgrade --quiet pip || return 1 if [[ -z $pkg_list ]]; then echo "${FUNCNAME[0]}(): WARNING: No packages to install" return 0 fi echo "${FUNCNAME[0]}(): INFO: Installing: $pkg_list" # $pkg_list is expected to be unquoted # shellcheck disable=SC2086 "$lf_venv/bin/pip" install --upgrade --quiet $pkg_list || return 1 ;; python3*) local pkg_list="" # Use pyenv for selecting the python version if [[ -d "/opt/pyenv" ]]; then echo "Setup pyenv:" export PYENV_ROOT="/opt/pyenv" export PATH="$PYENV_ROOT/bin:$PATH" pyenv versions if command -v pyenv 1>/dev/null 2>&1; then eval "$(pyenv init - --no-rehash)" # shellcheck disable=SC2046 pyenv local $(lf-pyver "${python}") fi fi # Add version specifier for some packages for arg in "$@"; do case $arg in jenkins-job-builder) pkg_list+="jenkins-job-builder==${JJB_VERSION:-6.3.0} " ;; *) pkg_list+="$arg " ;; esac done # Precedence: # - Re-use venv: # 1. --venv-file as lf_venv # 2. default: "/tmp/.os_lf_venv" # - Create new venv when 1. and 2. is absent if [[ -f "$venv_file" ]]; then lf_venv=$(cat "$venv_file") echo "${FUNCNAME[0]}(): INFO: Reuse venv:$lf_venv from" \ "file:$venv_file" elif [[ ! -f "$venv_file" ]]; then if [[ -n "$install_args" ]]; then $python -m venv "$install_args" "$lf_venv" || return 1 else $python -m venv "$lf_venv" || return 1 fi echo "${FUNCNAME[0]}(): INFO: Creating $python venv at $lf_venv" echo "$lf_venv" > "$venv_file" echo "${FUNCNAME[0]}(): INFO: Save venv in file: $venv_file" fi echo "${FUNCNAME[0]}(): INFO: Installing base packages (pip, setuptools, virtualenv)" # Define pip install options for network issues local pip_opts="--upgrade --quiet" # Add trusted hosts to bypass SSL certificate issues # This is the most common cause of the "Could not find a version" error pip_opts="$pip_opts --trusted-host pypi.org" pip_opts="$pip_opts --trusted-host files.pythonhosted.org" pip_opts="$pip_opts --trusted-host pypi.python.org" # If behind a corporate proxy, try to detect and handle it if [[ -n "${HTTP_PROXY:-}" || -n "${HTTPS_PROXY:-}" ]]; then echo "${FUNCNAME[0]}(): INFO: Proxy detected, using proxy settings" if [[ -n "${HTTP_PROXY:-}" ]]; then pip_opts="$pip_opts --proxy $HTTP_PROXY" elif [[ -n "${HTTPS_PROXY:-}" ]]; then pip_opts="$pip_opts --proxy $HTTPS_PROXY" fi fi # shellcheck disable=SC2086 if ! lf-venv-pip-install "$lf_venv/bin/python3" $pip_opts \ pip 'setuptools<66' virtualenv; then lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Cannot install base packages" lf-echo-stderr "The configured PyPI index and upstream PyPI both failed;" \ "check the PyPI mirror, firewall, proxy and DNS settings" return 1 fi # Continue with the rest of the package installation if [[ -z $pkg_list ]]; then echo "${FUNCNAME[0]}(): WARNING: No additional packages to install" else echo "${FUNCNAME[0]}(): INFO: Installing additional packages: $pkg_list" # $pkg_list is expected to be unquoted # shellcheck disable=SC2086 if ! lf-venv-pip-install "$lf_venv/bin/python3" $pip_opts \ --upgrade-strategy eager $pkg_list; then lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Failed to install packages: $pkg_list" return 1 fi fi ;; *) lf-echo-stderr "${FUNCNAME[0]}(): ERROR: No support for: $python" return 1 ;; esac if ! type "$python" > /dev/null; then lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Unknown Python: $python" return 1 fi if $set_path; then echo "${FUNCNAME[0]}(): INFO: Adding $lf_venv/bin to PATH" PATH=$lf_venv/bin:$PATH return 0 else echo "${FUNCNAME[0]}(): INFO: Path not set, lf_venv set to: $lf_venv" fi } # End lf-activate-venv ################################################################################ # # NAME # lf-git-validate-jira-urls # # SYNOPSIS # # shellcheck disable=SC1090 # source ~/lf-env.sh # # lf-git-validate-jira-urls # # DESCRIPTION # Check for JIRA URLS in the commit message # # RETURN VALUES # OK: 0 # Fail: 1 # ################################################################################ lf-git-validate-jira-urls () { echo "Checking for JIRA URLs in commit message..." # if JIRA_URL is not defined, nothing to do if [[ -v JIRA_URL ]]; then base_url=$(echo "$JIRA_URL" | awk -F'/' '{print $3}') jira_link=$(git rev-list --format=%B --max-count=1 HEAD | \ grep -io "http[s]*://$base_url/" || true) if [[ -n $jira_link ]]; then lf-echo-stderr \ "${FUNCNAME[0]}(): ERROR: JIRA URL found in commit message" lf-echo-stderr \ 'Add jira references as: Issue: -,'\ ' instead of URLs' return 1 fi else echo "${FUNCNAME[0]}(): WARNING: JIRA_URL not set, continuing anyway" fi return 0 } ################################################################################ # # NAME # lf-jjb-check-ascii # # SYNOPSIS # # shellcheck disable=SC1090 # source ~/lf-env.sh # # lf-jjb-check-ascii # # DESCRIPTION # Check for JJB YAML files containing non-printable ascii characters. This # function must be run from the top of the global-jjb repo. # # RETURN VALUES # OK: 0 # Fail: 1 # ################################################################################ lf-jjb-check-ascii () { if [[ ! -d "jjb" ]]; then lf-echo-stderr "${FUNCNAME[0]}(): ERROR: missing jjb directory" lf-echo-stderr \ "This function can only be run from top of global-jjb directory" return 1 fi if LC_ALL=C grep -I -r '[^[:print:][:space:]]' jjb/; then lf-echo-stderr \ "${FUNCNAME[0]}(): ERROR: Found YAML files containing"\ " non-printable characters." return 1 fi echo "${FUNCNAME[0]}(): INFO: All JJB YAML files contain only printable"\ " ASCII characters" return 0 } ################################################################################ # Functions that assign Variables ################################################################################ # These variables are shell (local) variables and need to be lower-case so # Shellcheck knows they are shell variables and will check for # 'used-before-set'. lf-set-maven-options () { # Disable 'unused-variable' check # shellcheck disable=SC2034 maven_options="--show-version --batch-mode -Djenkins \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.\ transfer.Slf4jMavenTransferListener=warn \ -Dmaven.repo.local=/tmp/r \ -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r" } ################################################################################ # # NAME # lf-pyver [python-version X.Y] # # SYNOPSIS # pyver 3.8 (outputs 3.8.13) # or # pyver 3.10 (outputs 3.10.6) # or # pyver 3 (outputs the most recent version 3.10.6) # # DESCRIPTION # The function takes short python version in the format and X.Y and prints # the semver format (X.Y.Z) of the version that has been installed on the system # with pyenv. # # When the expected version is not installed, nothing is returned. # # RETURN VALUES # OK: 0 # Fail: 1 # ################################################################################ lf-pyver() { local py_version_xy="${1:-python3}" local py_version_xyz="" pyenv versions | sed 's/^[ *]* //' | awk '{ print $1 }' | grep -E '^[0-9.]*[0-9]$' \ > "/tmp/.pyenv_versions" if [[ ! -s "/tmp/.pyenv_versions" ]]; then lf-echo-stderr "${FUNCNAME[0]}(): ERROR: pyenv not available" return 1 fi # strip out any prefix for (ex: 'python3.8' or 'v3.8') and match regex # to the output return by pyenv py_version_xyz=$(grep "^${py_version_xy//[a-zA-Z]/}" "/tmp/.pyenv_versions" | sort -V | tail -n 1;) if [[ -z ${py_version_xyz} ]]; then lf-echo-stderr "${FUNCNAME[0]}(): ERROR: Not installed on host: ${py_version_xy}" return 1 fi echo "${py_version_xyz}" return 0 } ################################################################################ # # NAME # lftools-uv [lftools-uv arguments]... # # SYNOPSIS # # shellcheck disable=SC1090 # source ~/lf-env.sh # # lftools-uv deploy nexus-zip "$NEXUS_URL" javadoc "$DEPLOY_PATH" javadoc.zip # or # lftools-uv nexus release --server "$NEXUS_URL" "$REPO" # # DESCRIPTION # Run lftools-uv through 'uvx', in an environment of its own: nothing is # installed into the job's Python or its virtualenvs, so the tool and its # dependencies cannot clash with a project's own. PYTHONPATH and PYTHONHOME # are cleared for uvx, so that modules a job puts on its own path cannot # shadow lftools-uv's; every other variable, credentials included, passes # through. # # The version is pinned: LFTOOLS_UV_VERSION overrides it, and # LFTOOLS_UV_PYTHON the Python version uvx runs it with. uvx downloads that # Python when the build agent does not have it. # # uvx is found on PATH or in ~/.local/bin, where uv-install.sh puts it, and # installed with the uv installer, honouring UV_VERSION, if neither has it. # A failed download or install, or a uvx that then does not run, is an # error. # # Packages come from the 'index-url' a build agent's pip.conf sets, or # PIP_INDEX_URL, else PyPI. While the mirror is unreachable, lftools-uv is # fetched from PyPI instead, as lf-venv-pip-install does for pip. Only that # fetch is retried: once it succeeds the command itself runs exactly once, # offline, from uv's cache. The fetch happens once per shell, which keeps # what it found in _lf_uvx_spec and _lf_uvx_index; a subshell, such as a # command substitution, inherits them but cannot keep them, so call # lftools-uv once in the main shell before calling it from many subshells. # # An index URL may carry credentials. Messages show it without them, and # the function turns off 'set -x' while it runs, restoring the caller's # setting on return, so that its trace cannot show them either. # # RETURN VALUES # The exit status of lftools-uv, or 1 when uv or lftools-uv cannot be # installed or fetched. # ################################################################################ lftools-uv () { local xtrace="" rc=0 if [[ $- == *x* ]]; then xtrace=1 set +x fi _lf-lftools-uv "$@" || rc=$? if [[ -n $xtrace ]]; then set -x fi return "$rc" } # Print URL $1 without the user:password it may carry. _lf-redact-url () { sed -E 's#^([[:alpha:]][[:alnum:]+.-]*://)[^/]*@#\1***@#' <<< "$1" } # Install uv into ~/.local/bin, as uv-install.sh does. Fail unless the # installer downloads and runs cleanly, and the uvx it leaves runs. _lf-install-uv () { local installer rc=0 lf-echo-stderr "lftools-uv: uvx not found; installing uv" if ! installer="$(mktemp /tmp/uv-install-XXXXXX.sh)"; then lf-echo-stderr "lftools-uv: ERROR: cannot create a file for the uv installer" return 1 fi if ! wget -nv -O "$installer" \ "https://astral.sh/uv/${UV_VERSION:+$UV_VERSION/}install.sh" >&2; then lf-echo-stderr "lftools-uv: ERROR: cannot download the uv installer" rc=1 elif ! sh "$installer" >&2; then lf-echo-stderr "lftools-uv: ERROR: the uv installer failed" rc=1 fi rm -f "$installer" if (( rc == 0 )) && ! ~/.local/bin/uvx --version > /dev/null; then lf-echo-stderr "lftools-uv: ERROR: uvx does not run after installing uv" rc=1 fi return "$rc" } _lf-lftools-uv () { local version="${LFTOOLS_UV_VERSION:-0.8.2}" local python="${LFTOOLS_UV_PYTHON:-3.12}" local spec="lftools-uv[openstack]==${version}" local upstream="https://pypi.org/simple" local uvx index local -a run uvx="$(command -v uvx || true)" if [[ -z $uvx && -x ~/.local/bin/uvx ]]; then uvx=~/.local/bin/uvx fi if [[ -z $uvx ]]; then _lf-install-uv || return 1 uvx=~/.local/bin/uvx fi run=(env -u PYTHONPATH -u PYTHONHOME "$uvx" --quiet --python "$python" --from "$spec") # Fetch the pinned version once per shell, from the build agent's mirror # or, while that is unreachable, from PyPI; remember which one worked. if [[ ${_lf_uvx_spec:-} != "$spec $python" ]]; then index="${PIP_INDEX_URL:-}" if [[ -z $index && -f ~/.config/pip/pip.conf ]]; then index="$(sed -n 's/^[[:space:]]*index-url[[:space:]]*=[[:space:]]*\([^[:space:]]*\).*/\1/p' \ ~/.config/pip/pip.conf | head -n 1)" fi index="${index:-$upstream}" if ! "${run[@]}" --default-index "$index" lftools-uv --version > /dev/null; then if [[ $index == "$upstream" ]]; then lf-echo-stderr "lftools-uv: ERROR: could not fetch $spec" return 1 fi lf-echo-stderr "lftools-uv: WARNING: could not fetch $spec from" \ "$(_lf-redact-url "$index"); retrying against $upstream" index="$upstream" if ! "${run[@]}" --default-index "$index" lftools-uv --version > /dev/null; then lf-echo-stderr "lftools-uv: ERROR: could not fetch $spec" return 1 fi fi _lf_uvx_spec="$spec $python" _lf_uvx_index="$index" fi "${run[@]}" --offline --default-index "$_lf_uvx_index" lftools-uv "$@" }