#!/bin/sh # This file can be used both as executable script or library to be sourced. # To use as library to be sourced, set DOWNLOADREPOS_AS_LIB=1 env var. downloadrepos() { github_repo="$1"; shift branch="$1"; shift wget --quiet --output-document - \ $github_repo/archive/${branch}.tar.gz | tar xz } downloadgithubrepos() { owner_and_repo_name="$1"; shift repo_owner="`echo "$owner_and_repo_name" | sed "s@/.*\\$@@g"`" repo_name="`echo "$owner_and_repo_name" | sed "s@^.*/@@g"`" repo_branch="$1"; shift # clean up other previously downloaded branches of the same repo as well # NOTE: # if DOWNLOAD_REPOS_SKIP_CLEANUP=true is used, # you most probably need TEST_ALL_SKIP_CLEANUP=true as well # otherwise, contents are reset early when running 'testall' if [ x"${DOWNLOAD_REPOS_SKIP_CLEANUP}" != x"true" ]; then rm -rf ${repo_name}-* else echo "Skip cleanup of [${repo_name}] local archives." fi ls | grep $repo_name if [ x"${DOWNLOAD_REPOS_SKIP_EXISTS}" = x"true" ] && [ -d "./${repo_name}-${repo_branch}" ]; then echo "Skip download [https://github.com/$repo_owner/$repo_name] [$repo_branch], already exists." else downloadrepos "https://github.com/$repo_owner/$repo_name" "$repo_branch" fi ls | grep $repo_name } # USAGE: VAR_TO_LOWER="$(lowercase "$VAR_TO_LOWER")" lowercase() { echo "$1" | tr '[:upper:]' '[:lower:]' } lowercase_boolean_build_params() { TEST_MAGPIE_AUTH="$(lowercase "$TEST_MAGPIE_AUTH")" TEST_PAVICS_SDI_REPO="$(lowercase "$TEST_PAVICS_SDI_REPO")" TEST_PAVICS_SDI_WEAVER="$(lowercase "$TEST_PAVICS_SDI_WEAVER")" TEST_FINCH_REPO="$(lowercase "$TEST_FINCH_REPO")" TEST_PAVICS_LANDING_REPO="$(lowercase "$TEST_PAVICS_LANDING_REPO")" TEST_RAVEN_REPO="$(lowercase "$TEST_RAVEN_REPO")" TEST_RAVENPY_REPO="$(lowercase "$TEST_RAVENPY_REPO")" TEST_ESGF_COMPUTE_API_REPO="$(lowercase "$TEST_ESGF_COMPUTE_API_REPO")" TEST_LOCAL_NOTEBOOKS="$(lowercase "$TEST_LOCAL_NOTEBOOKS")" } # Replace all slash (/) by dash (-) because (/) is illegal in folder name # for branch name of the format "feature/my_wizbang-feature". # Github does the same when downloading repo archive by downloadrepos above. # USAGE: export BRANCH_NAME="$(sanitize_branch_name "$BRANCH_NAME")" sanitize_branch_name() { echo "$1" | sed "s@/@-@g" } # Ex: extract 'pavics-sdi' from 'Ouranosinc/pavics-sdi'. # USAGE: REPO_NAME_ONLY="$(extract_repo_name "$REPO_NAME")" extract_repo_name() { echo "$1" | sed "s@^.*/@@g" } # Branches that have allowed characters such as '+' other than alphanum, '-', '_' and '.' are converted to '-' in archives. # USAGE: FOLDER_NAME="$(sanitize_extracted_folder_name "$FOLDER_NAME")" sanitize_extracted_folder_name() { echo "$1" | sed "s@[^a-zA-Z0-9_\-\.]@-@g" } downloadrepos_main() { . ./default_build_params lowercase_boolean_build_params if [ -z "$DOWNLOAD_ALL_DEFAULT_REPOS" ]; then # Back-compat with old default behavior, used in binder/reorg-notebook # and other external scripts that autodeploy tutorial notebooks (see # https://github.com/bird-house/birdhouse-deploy/blob/444a7c35a31aa8ad351e47f659383ba5c2919705/birdhouse/deployment/trigger-deploy-notebook#L64-L75) DOWNLOAD_ALL_DEFAULT_REPOS=true fi if [ -z "$1" ]; then if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_PAVICS_SDI_REPO" = xtrue ]; then downloadgithubrepos $PAVICS_SDI_REPO $PAVICS_SDI_BRANCH fi if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_FINCH_REPO" = xtrue ]; then downloadgithubrepos $FINCH_REPO $FINCH_BRANCH fi if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_PAVICS_LANDING_REPO" = xtrue ]; then downloadgithubrepos $PAVICS_LANDING_REPO $PAVICS_LANDING_BRANCH fi if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_RAVEN_REPO" = xtrue ]; then downloadgithubrepos $RAVEN_REPO $RAVEN_BRANCH fi if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_RAVENPY_REPO" = xtrue ]; then downloadgithubrepos $RAVENPY_REPO $RAVENPY_BRANCH fi if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_ESGF_COMPUTE_API_REPO" = xtrue ]; then downloadgithubrepos $ESGF_COMPUTE_API_REPO $ESGF_COMPUTE_API_BRANCH fi else set -x downloadrepos "$@" fi } # Choose artifact filename format under buildout/ dir that is archived by Jenkins. # # The corresponding .output.ipynb will also have the same filename format. # # Override this function in CONFIG_OVERRIDE_SCRIPT_URL to choose another # format, see demo in test-override/jenkins-params-external-repos.include.sh. # # Default implemenation: keep original file hierarchy. # # USAGE: artifact_filename="$(choose_artifact_filename "$original_nb_filename")" choose_artifact_filename() { echo "$1" } # Any post-processing steps at the end of runtest. # # Override this function in CONFIG_OVERRIDE_SCRIPT_URL to add extra steps, # see demo in test-override/jenkins-params-external-repos.include.sh. # post_runtest() { # Can not have empty function, have to put something here. echo "Default: no post_runtest() override." } # Allow full override of ALL Jenkins params before running test suite. # Intended to override all params in Jenkinsfile. # # This `CONFIG_PARAMETERS_SCRIPT_URL` is different than the # `CONFIG_OVERRIDE_SCRIPT_URL` in that it overrides all the # Jenkins params much sooner so all regular processing can take place. # # When the regular processing are not enough and we need additional # customizations, then we use the other `CONFIG_OVERRIDE_SCRIPT_URL`. # # The two override scripts complement each other by hooking into the # process at different moment in time, and together they allow full # complete override of the various options for the test run. # # Furthermore, CONFIG_OVERRIDE_SCRIPT_URL can also be a local file that is # created on the fly by CONFIG_PARAMETERS_SCRIPT_URL. So in the end, only # CONFIG_PARAMETERS_SCRIPT_URL is needed for a full complete override. # See example in test-override/jenkins-params-raven-specific-nb.include.sh. # # USAGE: handle_config_parameters_script_url "$CONFIG_PARAMETERS_SCRIPT_URL" # TMP_CONFIG_PARAMETERS_SCRIPT_URL="" # init before use handle_config_parameters_script_url() { TMP_CONFIG_PARAMETERS_SCRIPT_URL="$1" if [ -n "$TMP_CONFIG_PARAMETERS_SCRIPT_URL" ]; then TMP_PARAMS_OVERRIDE="/tmp/jenkins_params_override" rm -vf "$TMP_PARAMS_OVERRIDE" curl --fail --silent "$TMP_CONFIG_PARAMETERS_SCRIPT_URL" > "$TMP_PARAMS_OVERRIDE" CURL_EXIT_CODE=$? # Use cat to log content of override script for traceability. cat "$TMP_PARAMS_OVERRIDE" if [ $CURL_EXIT_CODE -eq 0 ]; then # Source script so it can alter ALL existing variables in the current context. . "$TMP_PARAMS_OVERRIDE" fi fi # Reset for subsequent runs. TMP_CONFIG_PARAMETERS_SCRIPT_URL="" } # Allow full override of ALL configs before running test suite. # # USAGE: handle_config_override_script_url "$CONFIG_OVERRIDE_SCRIPT_URL" # TMP_CONFIG_OVERRIDE_SCRIPT_URL="" # init before use handle_config_override_script_url() { TMP_CONFIG_OVERRIDE_SCRIPT_URL="$1" if [ -n "$TMP_CONFIG_OVERRIDE_SCRIPT_URL" ]; then TMP_CONF_OVERRIDE="/tmp/conf_override" rm -vf "$TMP_CONF_OVERRIDE" CURL_EXIT_CODE=0 if echo "$TMP_CONFIG_OVERRIDE_SCRIPT_URL" | grep -e ^http ; then curl --fail --silent "$TMP_CONFIG_OVERRIDE_SCRIPT_URL" > "$TMP_CONF_OVERRIDE" CURL_EXIT_CODE=$? else # Not starting with http, it's a local file. # Local file can be previously created by CONFIG_PARAMETERS_SCRIPT_URL. # See example in test-override/jenkins-params-raven-specific-nb.include.sh. TMP_CONF_OVERRIDE="$TMP_CONFIG_OVERRIDE_SCRIPT_URL" fi # Log content of override script for traceability. cat "$TMP_CONF_OVERRIDE" # Source script so it can alter ALL existing variables in the current context. if [ -f "$TMP_CONF_OVERRIDE" ] && [ $CURL_EXIT_CODE -eq 0 ]; then . "$TMP_CONF_OVERRIDE" fi fi # Reset for subsequent runs. TMP_CONFIG_OVERRIDE_SCRIPT_URL="" } # Any pre-processing steps before generating resulting notebooks. # # Override this function in CONFIG_OVERRIDE_SCRIPT_URL to alter # RESULTING_NOTEBOOKS. # pre_saving_resulting_nb() { # Can not have empty function, have to put something here. echo "Default: no pre_saving_resulting_nb() override." } # Check whether a notebook is enabled to produce output. Can be used to # blacklist a list of notebooks. # # Override this function in CONFIG_OVERRIDE_SCRIPT_URL. # # USAGE: enable_resulting_nb "