#!/bin/bash AMPLIFY_NODE_VERSION=24.12.0 # set exit on error to true set -e # The flags address the issue here: https://github.com/boto/botocore/issues/1716 export MSYS_NO_PATHCONV=1 export MSYS2_ARG_CONV_EXCL="*" # storeCache function storeCache { localPath="$1" alias="$2" environment="$3" s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" echo "Writing cache folder $alias to $s3Path" # zip contents and upload to s3 errorMessage="Something went wrong storing the cache folder $alias. Continuing anyway." # tar behaves differently on windows # Windows tar does not allow stdin/stdout Windows equivalent. # The archive needs to be written to a file first. # We don't also do this for Linux because: # 1. It is much slower. # 2. The linux version fails with `file changed as we read it`. # Branching the bash script is the easiest way around this if [[ $environment == "windows" ]]; then echo "Storing cache for Windows" if ! (cd $localPath && tar -czf cache.tar . && ls && aws s3 cp cache.tar $s3Path); then echo $errorMessage fi else echo "Storing cache for Linux" if ! (cd $localPath && tar cz . | aws s3 cp - $s3Path); then echo $errorMessage fi fi echo "Done writing cache folder $alias" cd $CODEBUILD_SRC_DIR } # loadCache function loadCache { alias="$1" localPath="$2" environment="$3" s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" echo "Loading cache folder from $s3Path" # create directory if it doesn't exist yet mkdir -p $localPath # check if cache exists in s3 if ! aws s3 ls $s3Path > /dev/null; then echo "Cache folder $alias not found." exit 0 fi # load cache and unzip it errorMessage="Something went wrong fetching the cache folder $alias. Continuing anyway." if [[ $environment == "windows" ]]; then # tar behaves differently on windows echo "Loading cache for Windows" if ! (cd $localPath && aws s3 cp $s3Path - | tar xzkf -); then echo $errorMessage fi else echo "Loading cache for Linux" if ! (cd $localPath && aws s3 cp $s3Path - | tar xz); then echo $errorMessage fi fi echo "Done loading cache folder $alias" cd $CODEBUILD_SRC_DIR } function storeCacheForLinuxBuildJob { # upload [repo, .cache] to s3 storeCache $CODEBUILD_SRC_DIR repo storeCache $HOME/.cache .cache } function storeCacheForWindowsBuildJob { storeCache $CODEBUILD_SRC_DIR repo-windows windows # Use LOCALAPPDATA if available, otherwise construct the path YARN_CACHE_PATH="${LOCALAPPDATA:-$HOME/AppData/Local}/Yarn/Cache/v6" storeCache "$YARN_CACHE_PATH" .cache-windows windows } function loadCacheFromLinuxBuildJob { # download [repo, .cache] from s3 loadCache repo $CODEBUILD_SRC_DIR loadCache .cache $HOME/.cache } function loadCacheFromWindowsBuildJob { # download [repo, .cache] from s3 loadCache repo-windows $CODEBUILD_SRC_DIR windows # Use LOCALAPPDATA if available, otherwise construct the path YARN_CACHE_PATH="${LOCALAPPDATA:-$HOME/AppData/Local}/Yarn/Cache/v6" loadCache .cache-windows "$YARN_CACHE_PATH" windows } function storeCacheFile { localFilePath="$1" alias="$2" s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" echo "Writing cache file $alias to $s3Path" # upload file to s3 if ! (aws s3 cp $localFilePath $s3Path); then echo "Something went wrong storing the cache file $alias." fi echo "Done writing cache file $alias" cd $CODEBUILD_SRC_DIR } function loadCacheFile { alias="$1" localFilePath="$2" s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" echo "Loading cache file $alias from $s3Path" # check if cache file exists in s3 if ! aws s3 ls $s3Path > /dev/null; then echo "Cache file $alias not found." exit 0 fi # load cache file if ! (aws s3 cp $s3Path $localFilePath); then echo "Something went wrong fetching the cache file $alias. Continuing anyway." fi echo "Done loading cache file $alias" cd $CODEBUILD_SRC_DIR } function _setShell { echo "Setting Shell" yarn config set script-shell $(which bash) } function _buildLinux { _setShell echo "Setup Node Version $AMPLIFY_NODE_VERSION for Linux" _setupNodeVersionLinux $AMPLIFY_NODE_VERSION echo "Linux Build" yarn run production-build storeCacheForLinuxBuildJob } function _buildWindows { echo "Setup Node Version $AMPLIFY_NODE_VERSION for Windows" _setupNodeVersionWindows $AMPLIFY_NODE_VERSION echo "Windows Build" yarn run production-build storeCacheForWindowsBuildJob } # used when build is not necessary for codebuild project function _installLinux { _setShell _setupNodeVersionLinux $AMPLIFY_NODE_VERSION echo "Linux Install" yarn run production-install storeCacheForLinuxBuildJob } function _testLinux { echo "Run Unit Test Linux" loadCacheFromLinuxBuildJob _setupNodeVersionLinux $AMPLIFY_NODE_VERSION yarn test-ci } function _testWindows { echo "Run Unit Test Windows" loadCacheFromWindowsBuildJob _setupNodeVersionWindows $AMPLIFY_NODE_VERSION yarn test-ci } function _verifyAPIExtract { echo "Verify API Extract" loadCacheFromLinuxBuildJob _setupNodeVersionLinux $AMPLIFY_NODE_VERSION yarn verify-api-extract } function _verifyDependencyLicensesExtract { echo "Verify Dependency Licenses Extract" loadCacheFromLinuxBuildJob _setupNodeVersionLinux $AMPLIFY_NODE_VERSION yarn verify-dependency-licenses-extract } function _lint { echo "Lint" loadCacheFromLinuxBuildJob _setupNodeVersionLinux $AMPLIFY_NODE_VERSION chmod +x .codebuild/scripts/lint_pr.sh && ./.codebuild/scripts/lint_pr.sh } function _publishToLocalRegistry { echo "Publish To Local Registry" loadCacheFromLinuxBuildJob _setupNodeVersionLinux $AMPLIFY_NODE_VERSION if [ -z "$BRANCH_NAME" ]; then if [ -z "$CODEBUILD_WEBHOOK_TRIGGER" ]; then export BRANCH_NAME="$(git symbolic-ref HEAD --short 2>/dev/null)" if [ "$BRANCH_NAME" = "" ] ; then BRANCH_NAME="$(git rev-parse HEAD | xargs git name-rev | cut -d' ' -f2 | sed 's/remotes\/origin\///g')"; fi elif [[ "$CODEBUILD_WEBHOOK_TRIGGER" == "pr/"* ]]; then export BRANCH_NAME=${CODEBUILD_WEBHOOK_BASE_REF##*/} fi fi echo $BRANCH_NAME git checkout $BRANCH_NAME # Fetching git tags from upstream # For forked repo only # Can be removed when using team account echo "fetching tags" git fetch --tags https://github.com/aws-amplify/amplify-codegen source .codebuild/scripts/local_publish_helpers.sh startLocalRegistry "$(pwd)/.codebuild/scripts/verdaccio.yaml" setNpmRegistryUrlToLocal git config user.email not@used.com git config user.name "Doesnt Matter" setNpmTag if [ -z $NPM_TAG ]; then yarn publish-to-verdaccio else yarn lerna publish --exact --dist-tag=latest --preid=$NPM_TAG --conventional-commits --conventional-prerelease --no-verify-access --yes --no-commit-hooks --no-push --no-git-tag-version fi unsetNpmRegistryUrl # copy [verdaccio-cache] to s3 storeCache $CODEBUILD_SRC_DIR/../verdaccio-cache verdaccio-cache } function _installCLIFromLocalRegistry { environment="$1" echo "Start verdaccio, install CLI" source .codebuild/scripts/local_publish_helpers.sh # absolute paths do not work with verdaccio on windows if [[ $environment == "windows" ]]; then echo "Starting local registry for Windows" startLocalRegistry .codebuild/scripts/verdaccio.yaml else echo "Starting local registry for Linux" startLocalRegistry "$(pwd)/.codebuild/scripts/verdaccio.yaml" fi setNpmRegistryUrlToLocal changeNpmGlobalPath # set longer timeout to avoid socket timeout error npm config set fetch-retries 5 npm config set fetch-timeout 600000 npm config set fetch-retry-mintimeout 30000 npm config set fetch-retry-maxtimeout 180000 npm config set maxsockets 1 npm install -g @aws-amplify/cli-internal echo "using Amplify CLI version: "$(amplify --version) npm list -g --depth=1 | grep -e '@aws-amplify/amplify-category-api' -e 'amplify-codegen' unsetNpmRegistryUrl } function _loadTestAccountCredentials { echo ASSUMING PARENT TEST ACCOUNT credentials session_id=$((1 + $RANDOM % 10000)) # Use longer time for parent account role creds=$(aws sts assume-role --role-arn $TEST_ACCOUNT_ROLE --role-session-name testSession${session_id} --duration-seconds 3600) if [ -z $(echo $creds | jq -c -r '.AssumedRoleUser.Arn') ]; then echo "Unable to assume parent e2e account role." return fi echo "Using account credentials for $(echo $creds | jq -c -r '.AssumedRoleUser.Arn')" export AWS_ACCESS_KEY_ID=$(echo $creds | jq -c -r ".Credentials.AccessKeyId") export AWS_SECRET_ACCESS_KEY=$(echo $creds | jq -c -r ".Credentials.SecretAccessKey") export AWS_SESSION_TOKEN=$(echo $creds | jq -c -r ".Credentials.SessionToken") } function _setupE2ETestsLinux { echo "Setup E2E Tests Linux" loadCacheFromLinuxBuildJob loadCache verdaccio-cache $CODEBUILD_SRC_DIR/../verdaccio-cache _setupNodeVersionLinux $AMPLIFY_NODE_VERSION _installCLIFromLocalRegistry _loadTestAccountCredentials _setShell } function _setupE2ETestsWindows { echo "Setup E2E Tests Windows" loadCacheFromWindowsBuildJob loadCache verdaccio-cache $CODEBUILD_SRC_DIR/../verdaccio-cache windows _setupNodeVersionWindows $AMPLIFY_NODE_VERSION _installCLIFromLocalRegistry windows _loadTestAccountCredentials _setShell } function _setupGen2E2ETestsLinux { echo "Setup Gen2 E2E Tests Linux" loadCacheFromLinuxBuildJob loadCache verdaccio-cache $CODEBUILD_SRC_DIR/../verdaccio-cache _setupNodeVersionLinux $AMPLIFY_NODE_VERSION _loadTestAccountCredentials _setShell } function _setupGen2E2ETestsWindows { echo "Setup Gen2 E2E Tests Windows" loadCacheFromWindowsBuildJob loadCache verdaccio-cache $CODEBUILD_SRC_DIR/../verdaccio-cache windows _setupNodeVersionWindows $AMPLIFY_NODE_VERSION _loadTestAccountCredentials _setShell } function _runE2ETestsLinux { echo "RUN E2E Tests Linux" retry runE2eTest } function _runE2ETestsWindows { echo "RUN E2E Tests Windows" retry runE2eTest } function _runGen2E2ETestsLinux { echo "RUN Gen2 E2E Tests Linux" echo "Setup Node Version" _setupNodeVersionLinux $AMPLIFY_NODE_VERSION retry runGen2E2eTest } function _runGen2E2ETestsWindows { echo "RUN Gen2 E2E Tests Windows" echo "Setup Node Version" _setupNodeVersionWindows $AMPLIFY_NODE_VERSION retry runGen2E2eTest } function _scanArtifacts { if ! npx ts-node .codebuild/scripts/scan_artifacts.ts; then echo "Cleaning the repository" git clean -fdx exit 1 fi } function _cleanupE2EResources { echo "Cleanup E2E resources" loadCacheFromLinuxBuildJob _setupNodeVersionLinux $AMPLIFY_NODE_VERSION cd packages/amplify-codegen-e2e-tests echo "Running clean up script" build_batch_arn=$(aws codebuild batch-get-builds --ids $CODEBUILD_BUILD_ID | jq -r -c '.builds[0].buildBatchArn') echo "Cleanup resources for batch build $build_batch_arn" yarn clean-e2e-resources buildBatchArn $build_batch_arn } function _unassumeTestAccountCredentials { echo "Unassume Role" unset AWS_ACCESS_KEY_ID unset AWS_SECRET_ACCESS_KEY unset AWS_SESSION_TOKEN } # The following functions are forked from circleci local publish helper # The e2e helper functions are moved for codebuild usage function useChildAccountCredentials { if [ -z "$USE_PARENT_ACCOUNT" ]; then export AWS_PAGER="" export AWS_MAX_ATTEMPTS=5 export AWS_STS_REGIONAL_ENDPOINTS=regional parent_acct=$(aws sts get-caller-identity | jq -cr '.Account') child_accts=$(aws organizations list-accounts | jq -c "[.Accounts[].Id | select(. != \"$parent_acct\")]") org_size=$(echo $child_accts | jq 'length') opt_in_regions=$(jq -r '.[] | select(.optIn == true) | .name' $CODEBUILD_SRC_DIR/scripts/e2e-test-regions.json) if echo "$opt_in_regions" | grep -qw "$CLI_REGION"; then child_accts=$(echo $child_accts | jq -cr '.[]') for child_acct in $child_accts; do # Get enabled opt-in regions for the child account enabled_regions=$(aws account list-regions --account-id $child_acct --region-opt-status-contains ENABLED) # Check if given opt-in region is enabled for the child account if echo "$enabled_regions" | jq -e ".Regions[].RegionName == \"$CLI_REGION\""; then pick_acct=$child_acct break fi done else pick_acct=$(echo $child_accts | jq -cr ".[$RANDOM % $org_size]") fi session_id=$((1 + $RANDOM % 10000)) if [[ -z "$pick_acct" || -z "$session_id" ]]; then echo "Unable to find a child account. Fatal error and test run aborted" exit 1 fi creds=$(aws sts assume-role --role-arn arn:aws:iam::${pick_acct}:role/OrganizationAccountAccessRole --role-session-name testSession${session_id} --duration-seconds 3600) if [ -z $(echo $creds | jq -c -r '.AssumedRoleUser.Arn') ]; then echo "Unable to assume child account role. Fatal error and test run aborted" exit 1 fi export ORGANIZATION_SIZE=$org_size export CREDS=$creds echo "Using account credentials for $(echo $creds | jq -c -r '.AssumedRoleUser.Arn')" export AWS_ACCESS_KEY_ID=$(echo $creds | jq -c -r ".Credentials.AccessKeyId") export AWS_SECRET_ACCESS_KEY=$(echo $creds | jq -c -r ".Credentials.SecretAccessKey") export AWS_SESSION_TOKEN=$(echo $creds | jq -c -r ".Credentials.SessionToken") else echo "Using parent account credentials." fi echo "Region is set to use $CLI_REGION" } function retry { MAX_ATTEMPTS=2 SLEEP_DURATION=5 FIRST_RUN=true RUN_INDEX=0 FAILED_TEST_REGEX_FILE="./amplify-e2e-reports/amplify-e2e-failed-test.txt" if [ -f $FAILED_TEST_REGEX_FILE ]; then rm -f $FAILED_TEST_REGEX_FILE fi until [ $RUN_INDEX -ge $MAX_ATTEMPTS ] do echo "Attempting $@ with max retries $MAX_ATTEMPTS" setAwsAccountCredentials RUN_INDEX="$RUN_INDEX" "$@" && break RUN_INDEX=$[$RUN_INDEX+1] FIRST_RUN=false echo "Attempt $RUN_INDEX completed." sleep $SLEEP_DURATION done if [ $RUN_INDEX -ge $MAX_ATTEMPTS ]; then echo "failed: ${@}" >&2 exit 1 fi resetAwsAccountCredentials TEST_SUITE=${TEST_SUITE:-"TestSuiteNotSet"} echo "Attempt $RUN_INDEX succeeded." exit 0 # don't fail the step if putting the metric fails } function resetAwsAccountCredentials { if [ -z "$AWS_ACCESS_KEY_ID_ORIG" ]; then echo "AWS Access Key environment variable is already set" else export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_ORIG fi if [ -z "$AWS_SECRET_ACCESS_KEY_ORIG" ]; then echo "AWS Secret Access Key environment variable is already set" else export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_ORIG fi if [ -z "$AWS_SESSION_TOKEN_ORIG" ]; then echo "AWS Session Token environment variable is already set" else export AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN_ORIG fi } function setAwsAccountCredentials { resetAwsAccountCredentials export AWS_ACCESS_KEY_ID_ORIG=$AWS_ACCESS_KEY_ID export AWS_SECRET_ACCESS_KEY_ORIG=$AWS_SECRET_ACCESS_KEY export AWS_SESSION_TOKEN_ORIG=$AWS_SESSION_TOKEN if [[ "$OSTYPE" == "msys" ]]; then # windows provided by circleci has this OSTYPE useChildAccountCredentials else echo "OSTYPE is $OSTYPE" curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip -o awscliv2.zip >/dev/null export PATH=$PATH:$(pwd)/aws/dist useChildAccountCredentials fi } function runE2eTest { FAILED_TEST_REGEX_FILE="./amplify-e2e-reports/amplify-e2e-failed-test.txt" if [ -z "$FIRST_RUN" ] || [ "$FIRST_RUN" == "true" ]; then echo "using Amplify CLI version: "$(amplify --version) cd $(pwd)/packages/amplify-codegen-e2e-tests fi if [ -f $FAILED_TEST_REGEX_FILE ]; then # read the content of failed tests failedTests=$(<$FAILED_TEST_REGEX_FILE) npm run e2e --maxWorkers=4 $TEST_SUITE -t "$failedTests" else npm run e2e --maxWorkers=4 $TEST_SUITE fi } function runGen2E2eTest { FAILED_TEST_REGEX_FILE="./amplify-e2e-reports/amplify-e2e-failed-test.txt" if [ -z "$FIRST_RUN" ] || [ "$FIRST_RUN" == "true" ]; then cd $(pwd)/packages/amplify-codegen-e2e-tests fi if [ -f $FAILED_TEST_REGEX_FILE ]; then # read the content of failed tests failedTests=$(<$FAILED_TEST_REGEX_FILE) npm run e2e-gen2 --maxWorkers=4 $TEST_SUITE -t "$failedTests" else npm run e2e-gen2 --maxWorkers=4 $TEST_SUITE fi } function _deploy { _setShell _setupNodeVersionLinux $AMPLIFY_NODE_VERSION echo "Deploy" echo "Authenticate with NPM" PUBLISH_TOKEN=$(echo "$NPM_PUBLISH_TOKEN" | jq -r '.token') echo "//registry.npmjs.org/:_authToken=$PUBLISH_TOKEN" > ~/.npmrc ./.codebuild/scripts/publish.sh } function _deprecate { loadCacheFromLinuxBuildJob _setupNodeVersionLinux $AMPLIFY_NODE_VERSION echo "Deprecate" echo "creating private package manifest" ./scripts/create-private-package-manifest.sh echo "Authenticate with NPM" if [ "$USE_NPM_REGISTRY" == "true" ]; then PUBLISH_TOKEN=$(echo "$NPM_PUBLISH_TOKEN" | jq -r '.token') echo "//registry.npmjs.org/:_authToken=$PUBLISH_TOKEN" > ~/.npmrc else yarn verdaccio-clean source .codebuild/scripts/local_publish_helpers.sh startLocalRegistry "$(pwd)/.codebuild/scripts/verdaccio.yaml" setNpmRegistryUrlToLocal fi yarn deprecate unsetNpmRegistryUrl } function _emitCodegenCanaryMetric { aws cloudwatch \ put-metric-data \ --metric-name CodegenCanarySuccessRate \ --namespace amplify-codegen-canary-e2e-tests \ --unit Count \ --value $CODEBUILD_BUILD_SUCCEEDING \ --dimensions branch=release,test=$(basename "$TEST_SUITE" .test.ts | sed "s/build-app-//") \ --region us-west-2 } function _emitRegionalizedCanaryMetric { aws cloudwatch \ put-metric-data \ --metric-name $CANARY_METRIC_NAME \ --namespace amplify-codegen-canary-e2e-tests \ --unit Count \ --value $CODEBUILD_BUILD_SUCCEEDING \ --dimensions branch=release,region=$CLI_REGION \ --region us-west-2 } function _setupNodeVersionLinux { local version=$1 # Version number passed as an argument echo "Installing NVM and setting Node.js version to $version" # Install NVM curl -o - https://raw.githubusercontent.com/nvm-sh/nvm/master/install.sh | bash # Load NVM export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # Install and use the specified Node.js version nvm install "$version" nvm use "$version" # Refresh environment variables source ~/.bashrc # Or source the appropriate shell profile file like ~/.bash_profile or ~/.zshrc # Verify the Node.js version in use echo "Node.js version in use:" node -v } function _setupNodeVersionWindows { local version=$1 # Version number passed as an argument echo "Installing Node.js version $version on Windows" # Install Node.js using Chocolatey echo "Installing Node.js version $version using Chocolatey" choco install -fy nodejs-lts --version=$version # Refresh environment variables echo "Refreshing environment variables" export PATH=$PATH # Verify the Node.js version in use nodeVersion=$(node -v) echo "Node version: $nodeVersion" }