#!/bin/bash -e clone_args="${clone_args:-"--skip-chained-deps --parental-inheritance --within-instance"}" usage() { cat << EOF Usage: openqa-clone-custom-git-refspec [CUSTOM_TEST_VAR_1=foo] [CUSTOM_TEST_VAR_2=bar] ... openqa-clone-custom-git-refspec [CUSTOM_TEST_VAR_1=foo] [CUSTOM_TEST_VAR_2=bar] ... Options: -v, --verbose execute with verbose output -h, -?, --help display this help -n, --dry-run execute in dry-run mode, do not clone any openQA jobs -c --clone-job-args pass additional parameters to 'openqa-clone-job', e.g. '--clone-job-args="--show-progress"'. The default parameters are '$clone_args' defined in the env variable \$clone_args. Examples: openqa-clone-custom-git-refspec https://github.com/os-autoinst/os-autoinst-distri-opensuse/pull/6529 https://openqa.opensuse.org/tests/835060 DESKTOP=textmode openqa-clone-custom-git-refspec https://github.com/coolgw/os-autoinst-distri-opensuse/tree/nfs https://openqa.opensuse.org/tests/835060 DESKTOP=textmode openqa-clone-custom-git-refspec -n -c '--show-progress' https://github.com/coolgw/os-autoinst-distri-opensuse/tree/nfs https://openqa.opensuse.org/tests/835060 DESKTOP=textmode openqa-clone-custom-git-refspec https://github.com/foursixnine/os-autoinst-distri-opensuse/tree/oopsitsbrokenagain https://openqa.opensuse.org/tests/3128467 NEEDLES_DIR='https://github.com/foursixnine/os-autoinst-needles-opensuse.git#oopsitsbrokenagain' To use authenticated requests, you need to either provide a GitHub token by adding 'GITHUB_TOKEN=...' to a configuration file at 'github-token.conf' in your $OPENQA_CONFIG folder or set the GITHUB_TOKEN environment variable. Use this snippet in your 'github-token.conf' to do so: GITHUB_TOKEN="YourTokenHere12345" EOF exit "$1" } set -o pipefail fail() { echo "$*" >&2 exit 1 } throw_json_error() { echo "in contents queried from $1:" echo "$2" exit 2 } extract_urls_from_pr() { urls=$(echo "$pr_content" | jq -r '.body | capture("^(?.*@openqa: Clone )(?http.*[0-9]*)") | .url') if [[ $urls == *"http"* ]]; then echo "$urls" fi } config_file="${OPENQA_CONFIG}/github-token.conf" get_token_from_file() { grep "^GITHUB_TOKEN=" "$config_file" | cut -d'"' -f2 } if [[ -n "$GITHUB_TOKEN" ]]; then AUTHENTICATED_REQUEST=" -u $GITHUB_TOKEN:x-oauth-basic" echo "GitHub OAuth token provided in environment variable, performing authenticated requests." elif [[ -f $config_file ]]; then AUTHENTICATED_REQUEST=" -u $(get_token_from_file):x-oauth-basic" echo "GitHub OAuth token found in config file, performing authenticated requests." fi curl_github="${curl_github:-"curl${AUTHENTICATED_REQUEST}"}" curl_openqa="${curl_openqa:-"curl"}" opts=$(getopt -o vhnc: --long verbose,dry-run,help,clone-job-args: -n "$0" -- "$@") || usage 1 eval set -- "$opts" while true; do case "$1" in -v | --verbose) set -x shift ;; -h | --help) usage 0 ;; -n | --dry-run) dry_run=true shift ;; -c | --clone-job-args) clone_args="$2 $clone_args" shift 2 ;; --) shift break ;; *) break ;; esac done job_list="${job:-"$2"}" if [[ -z "$repo_name" ]] || [[ -z "$pr" ]]; then first_arg="${1:?"Need 'url' as parameter pointing to a either a git branch, e.g. 'https://github.com/yourname/os-autoinst-distri-opensuse/tree/poo-12345', or a github pull request in the form of pull request url or a combination of 'repo_name' (sending repo) and 'pr' variables, e.g. either 'https://github.com/os-autoinst/os-autoinst-distri-opensuse/pull/1234' 'me/os-autoinst-distri-opensuse' and '1234' or 'branch'"}" if [[ $first_arg == *"pull"* ]]; then pr_url=$first_arg target_repo_part=${pr_url%%/pull*} pr="${pr_url##*pull/}" pr="${pr%%/*}" elif [[ $first_arg == *"tree"* ]]; then #maybe a branch_url is given branch_url=$first_arg forked_repo_part=${branch_url%%/tree*} branch=${branch_url##*tree/} repo_name=${forked_repo_part##*github.com/} casedir="${casedir:-"${forked_repo_part}.git#${branch}"}" build="${build:-"$repo_name#$branch"}" fi fi if [[ -z "$branch" ]] || [[ -z "$repo_name" ]]; then pr_url=${target_repo_part/github.com/api.github.com/repos}/pulls/$pr pr_content=$(eval "${curl_github} -s $pr_url") label=$(echo "$pr_content" | jq -r '.head.label') || throw_json_error "$pr_url" "$pr_content" if [[ "${label}" == "null" ]]; then echo "Github API rate limit might have been exceeded. If this is the case, generate one at https://github.com/settings/tokens and then export it as an environment variable" echo 'export GITHUB_TOKEN=foobar' echo "Github reply: " "$(echo "${pr_content}" | jq -r '.message')" exit 1 fi repo_name="${repo_name:-"${label%:*}/${target_repo_part##*/}"}" branch="${branch:-"${label##*:}"}" repo="${repo:-"https://github.com/${repo_name}.git"}" pr_urls=$(extract_urls_from_pr "$pr_content") job_list=${pr_urls:-"$job_list"} fi clone_job() { if [[ -z "$host" ]] || [[ -z "$job" ]]; then local job_url="${1:?"Need 'job_url' parameter"}" local host=${job_url%%/t*} local job=${job_url##*/} fi if [[ -z "$testsuite" ]] || [[ -z "$needles_dir" ]] || [[ -z "$productdir" ]]; then local json_url=${host}/tests/${job}/file/vars.json local json_data json_data=$(eval "${curl_openqa} -s ${json_url}") echo "$json_data" | jq . &> /dev/null \ || fail "Unreadable openQA job or no valid JSON data encountered. \ Please try 'curl $json_url' or select another job, e.g. in the same scenario: $host/t$job#next_previous" local testsuite="${testsuite:-"$(echo "$json_data" | jq -r '.TEST')"}" || throw_json_error "$json_url" "$json_data" local old_productdir old_productdir=$(echo "$json_data" | jq -r '.PRODUCTDIR') || throw_json_error "$json_url" "$json_data" local old_casedir old_casedir=$(echo "$json_data" | jq -r '.CASEDIR') || throw_json_error "$json_url" "$json_data" # set a proper productdir pointing to the checked out repository for # absolute+relative paths if [[ ${old_productdir:0:1} == "/" ]]; then local productdir="${productdir:-"${repo_name##*/}/${old_productdir##"$old_casedir"}"}" else local productdir="${productdir:-"${repo_name##*/}/${old_productdir#*"${old_casedir##*/}"}"}" fi productdir="${productdir/\/\//\/}" # avoid consecutive slashes local old_needledir old_needledir=$(echo "$json_data" | jq -r '.NEEDLES_DIR | select (.!=null)') || throw_json_error "$json_url" "$json_data" local needles_dir="${needles_dir:-"$old_needledir"}" needles_dir="${needles_dir:-"$old_productdir/needles"}" fi local repo_branch="${repo_branch:-"$repo_name#$branch"}" local test_suffix="${test_suffix:-"@$repo_branch"}" local build="${build:-"$repo_name#$pr"}" local casedir="${casedir:-"$repo#$branch"}" local GROUP="${GROUP:-0}" local dry_run="${dry_run:-""}" local scriptdir scriptdir=$(dirname "${BASH_SOURCE[0]}") local cmd="$dry_run $scriptdir/openqa-clone-job $clone_args \"$host\" \"$job\" _GROUP=\"$GROUP\" TEST+=\"$test_suffix\" BUILD=\"$build\" CASEDIR=\"$casedir\" PRODUCTDIR=\"$productdir\" NEEDLES_DIR=\"$needles_dir\"" [[ ${#args[@]} -ne 0 ]] && cmd=$cmd"$(printf " '%s'" "${args[@]}")" if [[ -n "$MARKDOWN" ]]; then eval "$cmd" | sed 's/^Created job.*: \([^ ]*\) -> \(.*\)$/* [\1](\2)/' else eval "$cmd" fi } if [[ -z "$host" ]] && [[ -z "$job_list" ]]; then echo "Need 'job_url' as parameter pointing to the openQA job to clone or 'host' and 'job' variables, e.g. either 'https://openqa.opensuse.org/tests/123456' or 'https://openqa.opensuse.org' and '123456'. Argument can also be a comma-separated list of job URLs or a single host and multiple ids." exit 1 fi args=("${@:3}") IFS=',' for i in $job_list; do clone_job "$i" done