#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # ============================================================================= # Apache Auron Project Build Script # Description: # This script automates the build process for the Apache Auron project. # ============================================================================= # ----------------------------------------------------------------------------- # Section: Supported versions constants # Description: # Define constants for supported component versions # ----------------------------------------------------------------------------- SUPPORTED_OS_IMAGES=("centos7" "ubuntu24" "rockylinux8" "debian11" "azurelinux3") SUPPORTED_SPARK_VERSIONS=("3.0" "3.1" "3.2" "3.3" "3.4" "3.5" "4.0" "4.1" "4.2") SUPPORTED_SCALA_VERSIONS=("2.12" "2.13") SUPPORTED_CELEBORN_VERSIONS=("0.5" "0.6") # Currently only one supported version, but kept plural for consistency SUPPORTED_UNIFFLE_VERSIONS=("0.10") SUPPORTED_PAIMON_VERSIONS=("1.2") SUPPORTED_FLINK_VERSIONS=("1.18") SUPPORTED_ICEBERG_VERSIONS=("1.10.1") SUPPORTED_HUDI_VERSIONS=("0.15") # ----------------------------------------------------------------------------- # Function: print_help # Description: # Print script usage information, supported options, and example commands. # ----------------------------------------------------------------------------- print_help() { echo "Usage: $0 [OPTIONS] " echo "Build Auron project with specified Maven profiles" echo echo "Options:" echo " --pre Activate pre-release profile" echo " --release Activate release profile" echo " --clean Clean before build (default: true)" echo " --skiptests Skip unit tests (default: true)" echo " --sparktests Run spark tests (default: false)" echo " --docker Build in Docker environment (default: false)" echo " --threads Maven build threads (e.g. 1, 4, 1C). Default: local unset, docker 8" IFS=','; echo " --image Docker image to use (e.g. ${SUPPORTED_OS_IMAGES[*]}, default: ${SUPPORTED_OS_IMAGES[*]:0:1})"; unset IFS IFS=','; echo " --sparkver Specify Spark version (e.g. ${SUPPORTED_SPARK_VERSIONS[*]})"; unset IFS IFS=','; echo " --flinkver Specify Flink version (e.g. ${SUPPORTED_FLINK_VERSIONS[*]})"; unset IFS IFS=','; echo " --scalaver Specify Scala version (e.g. ${SUPPORTED_SCALA_VERSIONS[*]})"; unset IFS IFS=','; echo " --celeborn Specify Celeborn version (e.g. ${SUPPORTED_CELEBORN_VERSIONS[*]})"; unset IFS IFS=','; echo " --uniffle Specify Uniffle version (e.g. ${SUPPORTED_UNIFFLE_VERSIONS[*]})"; unset IFS IFS=','; echo " --paimon Specify Paimon version (e.g. ${SUPPORTED_PAIMON_VERSIONS[*]})"; unset IFS IFS=','; echo " --iceberg Specify Iceberg version (e.g. ${SUPPORTED_ICEBERG_VERSIONS[*]})"; unset IFS IFS=','; echo " --hudi Specify Hudi version (e.g. ${SUPPORTED_HUDI_VERSIONS[*]})"; unset IFS echo " -h, --help Show this help message" echo echo "Environment variables:" echo " AURON_BUILD_BRANCH Override detected Git branch in generated build info" echo " AURON_BUILD_REVISION Override detected Git revision in generated build info" echo echo "Examples:" echo " $0 --pre --sparkver ${SUPPORTED_SPARK_VERSIONS[*]: -1}" \ "--scalaver ${SUPPORTED_SCALA_VERSIONS[*]: -1} -DskipBuildNative" echo " # Spark 3.5 with all supported third-party integrations" echo " $0 --docker true --image rockylinux8" \ "--clean true --skiptests true --release" \ "--sparkver 3.5 --scalaver 2.12" \ "--flinkver 1.18" \ "--celeborn 0.6" \ "--uniffle 0.10" \ "--paimon 1.2" \ "--iceberg 1.10.1" \ "--hudi 0.15" echo " # Spark 4.0 (Hudi is not supported)" echo " $0 --docker true --image rockylinux8" \ "--clean true --skiptests true --release" \ "--sparkver 4.0 --scalaver 2.13" \ "--flinkver 1.18" \ "--celeborn 0.6" \ "--uniffle 0.10" \ "--paimon 1.2" \ "--iceberg 1.10.1" echo " # Spark 4.1 (Iceberg and Hudi are not supported)" echo " $0 --docker true --image rockylinux8" \ "--clean true --skiptests true --release" \ "--sparkver 4.1 --scalaver 2.13" \ "--flinkver 1.18" \ "--celeborn 0.6" \ "--uniffle 0.10" \ "--paimon 1.2" exit 0 } # ----------------------------------------------------------------------------- # Function: check_supported_version # Description: # Validate if a provided version string exists in the supported version list. # ----------------------------------------------------------------------------- check_supported_version() { local value="$1" shift local arr=("$@") for v in "${arr[@]}"; do [[ "$v" == "$value" ]] && return 0 done return 1 } # ----------------------------------------------------------------------------- # Function: print_invalid_option_error # Description: # Print formatted error message for unsupported version or option. # ----------------------------------------------------------------------------- print_invalid_option_error() { local component="$1" local invalid_value="$2" shift 2 local supported_list supported_list=$(IFS=','; echo "$*") echo "ERROR: Unsupported $component: $invalid_value, currently supported: ${supported_list}" >&2 exit 1 } run_docker_compose_up() { local compose_args=("-f" "dev/docker-build/docker-compose.yml" "up" "--abort-on-container-exit") if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then echo "[INFO] Using Docker Compose command: docker compose" exec docker compose "${compose_args[@]}" fi if command -v docker-compose >/dev/null 2>&1; then echo "[INFO] Using Docker Compose command: docker-compose" exec docker-compose "${compose_args[@]}" fi echo "ERROR: Docker Compose is required. Install docker compose (Docker Engine v2) or legacy docker-compose." >&2 exit 1 } MVN_CMD="$(dirname "$0")/build/mvn" # ----------------------------------------------------------------------------- # Section: Initialize Variables # Description: # Initialize basic variables and default options for the build # ----------------------------------------------------------------------------- USE_DOCKER=false IMAGE_NAME="${SUPPORTED_OS_IMAGES[*]:0:1}" PRE_PROFILE=false RELEASE_PROFILE=false CLEAN=true SKIP_TESTS=true SPARK_TESTS=false THREADS="" SPARK_VER="" FLINK_VER="" SCALA_VER="" CELEBORN_VER="" UNIFFLE_VER="" PAIMON_VER="" ICEBERG_VER="" HUDI_VER="" MVN_D_ARGS=() # ----------------------------------------------------------------------------- # Section: Argument Parsing # Description: # Parse command-line options and validate their values. # ----------------------------------------------------------------------------- while [[ $# -gt 0 ]]; do case "$1" in --pre) PRE_PROFILE=true shift ;; --release) RELEASE_PROFILE=true shift ;; --docker) if [[ -n "$2" && "$2" =~ ^(true|false)$ ]]; then USE_DOCKER="$2" shift 2 else echo "ERROR: --docker requires true/false" >&2 exit 1 fi ;; --threads) if [[ -n "$2" && "$2" != -* ]]; then THREADS="$2" # Validate THREADS to prevent command injection when passed through Docker # Maven -T accepts an integer, optionally followed by 'C' (e.g., 4, 8, 2C) if [[ ! "$THREADS" =~ ^[0-9]+C?$ ]]; then echo "ERROR: Invalid --threads value '$THREADS'. Expected digits with optional 'C' (e.g., 4, 8, 2C)." >&2 exit 1 fi shift 2 else echo "ERROR: --threads requires a value (e.g. 1, 4, 1C)" >&2 exit 1 fi ;; --image) if [[ -n "$2" && "$2" != -* ]]; then IMAGE_NAME="$2" if ! check_supported_version "$IMAGE_NAME" "${SUPPORTED_OS_IMAGES[@]}"; then print_invalid_option_error Image "$IMAGE_NAME" "${SUPPORTED_OS_IMAGES[@]}" fi shift 2 else IFS=','; echo "ERROR: Missing argument for --image," \ "specify one of: ${SUPPORTED_OS_IMAGES[*]}" >&2; unset IFS exit 1 fi ;; --clean) if [[ -n "$2" && "$2" =~ ^(true|false)$ ]]; then CLEAN="$2" shift 2 else echo "ERROR: --clean requires true/false" >&2 exit 1 fi ;; --skiptests) if [[ -n "$2" && "$2" =~ ^(true|false)$ ]]; then SKIP_TESTS="$2" shift 2 else echo "ERROR: --skiptests requires true/false" >&2 exit 1 fi ;; --sparktests) if [[ -n "$2" && "$2" =~ ^(true|false)$ ]]; then SPARK_TESTS="$2" shift 2 else echo "ERROR: --sparktests requires true/false" >&2 exit 1 fi ;; --sparkver) if [[ -n "$2" && "$2" != -* ]]; then SPARK_VER="$2" if ! check_supported_version "$SPARK_VER" "${SUPPORTED_SPARK_VERSIONS[@]}"; then print_invalid_option_error Spark "$SPARK_VER" "${SUPPORTED_SPARK_VERSIONS[@]}" fi shift 2 else IFS=','; echo "ERROR: Missing argument for --sparkver," \ "specify one of: ${SUPPORTED_SPARK_VERSIONS[*]}" >&2; unset IFS exit 1 fi ;; --scalaver) if [[ -n "$2" && "$2" != -* ]]; then SCALA_VER="$2" if ! check_supported_version "$SCALA_VER" "${SUPPORTED_SCALA_VERSIONS[@]}"; then print_invalid_option_error Scala "$SCALA_VER" "${SUPPORTED_SCALA_VERSIONS[@]}" fi shift 2 else IFS=','; echo "ERROR: Missing argument for --scalaver," \ "specify one of: ${SUPPORTED_SCALA_VERSIONS[*]}" >&2; unset IFS exit 1 fi ;; --celeborn) if [[ -n "$2" && "$2" != -* ]]; then CELEBORN_VER="$2" if ! check_supported_version "$CELEBORN_VER" "${SUPPORTED_CELEBORN_VERSIONS[@]}"; then print_invalid_option_error Celeborn "$CELEBORN_VER" "${SUPPORTED_CELEBORN_VERSIONS[@]}" fi shift 2 else IFS=','; echo "ERROR: Missing argument for --celeborn," \ "specify one of: ${SUPPORTED_CELEBORN_VERSIONS[*]}" >&2; unset IFS exit 1 fi ;; --uniffle) if [[ -n "$2" && "$2" != -* ]]; then UNIFFLE_VER="$2" if ! check_supported_version "$UNIFFLE_VER" "${SUPPORTED_UNIFFLE_VERSIONS[@]}"; then print_invalid_option_error Uniffle "$UNIFFLE_VER" "${SUPPORTED_UNIFFLE_VERSIONS[@]}" fi shift 2 else IFS=','; echo "ERROR: Missing argument for --uniffle," \ "specify one of: ${SUPPORTED_UNIFFLE_VERSIONS[*]}" >&2; unset IFS exit 1 fi ;; --paimon) if [[ -n "$2" && "$2" != -* ]]; then PAIMON_VER="$2" if ! check_supported_version "$PAIMON_VER" "${SUPPORTED_PAIMON_VERSIONS[@]}"; then print_invalid_option_error Paimon "$PAIMON_VER" "${SUPPORTED_PAIMON_VERSIONS[@]}" fi shift 2 else IFS=','; echo "ERROR: Missing argument for --paimon," \ "specify one of: ${SUPPORTED_PAIMON_VERSIONS[*]}" >&2; unset IFS exit 1 fi ;; --iceberg) if [[ -n "$2" && "$2" != -* ]]; then ICEBERG_VER="$2" if ! check_supported_version "$ICEBERG_VER" "${SUPPORTED_ICEBERG_VERSIONS[@]}"; then print_invalid_option_error Iceberg "$ICEBERG_VER" "${SUPPORTED_ICEBERG_VERSIONS[@]}" fi if [ -z "$SPARK_VER" ]; then echo "ERROR: Building iceberg requires spark at the same time, and only Spark versions 3.4 to 4.0 are supported." exit 1 fi if [ "$SPARK_VER" != "3.4" ] && [ "$SPARK_VER" != "3.5" ] && [ "$SPARK_VER" != "4.0" ]; then echo "ERROR: Building iceberg requires spark versions are 3.4, 3.5 or 4.0." exit 1 fi shift 2 else IFS=','; echo "ERROR: Missing argument for --iceberg," \ "specify one of: ${SUPPORTED_ICEBERG_VERSIONS[*]}" >&2; unset IFS exit 1 fi ;; --hudi) if [[ -n "$2" && "$2" != -* ]]; then HUDI_VER="$2" if ! check_supported_version "$HUDI_VER" "${SUPPORTED_HUDI_VERSIONS[@]}"; then print_invalid_option_error Hudi "$HUDI_VER" "${SUPPORTED_HUDI_VERSIONS[@]}" fi if [ -z "$SPARK_VER" ]; then echo "ERROR: Building hudi requires spark at the same time, and only Spark versions 3.0 to 3.5 are supported." exit 1 fi if [ "$SPARK_VER" != "3.0" ] && [ "$SPARK_VER" != "3.1" ] && [ "$SPARK_VER" != "3.2" ] && [ "$SPARK_VER" != "3.3" ] && [ "$SPARK_VER" != "3.4" ] && [ "$SPARK_VER" != "3.5" ]; then echo "ERROR: Building hudi requires spark versions are 3.0 to 3.5." exit 1 fi shift 2 else IFS=','; echo "ERROR: Missing argument for --hudi," \ "specify one of: ${SUPPORTED_HUDI_VERSIONS[*]}" >&2; unset IFS exit 1 fi ;; --flinkver) if [[ -n "$2" && "$2" != -* ]]; then FLINK_VER="$2" if ! check_supported_version "$FLINK_VER" "${SUPPORTED_FLINK_VERSIONS[@]}"; then print_invalid_option_error Flink "$FLINK_VER" "${SUPPORTED_FLINK_VERSIONS[@]}" fi shift 2 else IFS=','; echo "ERROR: Missing argument for --flink," \ "specify one of: ${SUPPORTED_FLINK_VERSIONS[*]}" >&2; unset IFS exit 1 fi ;; -h|--help) print_help ;; --*) echo "ERROR: Unknown option '$1'" >&2 echo "Use '$0 --help' for usage information" >&2 exit 1 ;; -*) if [[ "$1" == -D* ]]; then MVN_D_ARGS+=("$1") shift else break fi ;; *) echo "ERROR: $1 is not supported" >&2 echo "Use '$0 --help' for usage information" >&2 exit 1 ;; esac done # Detect conflict between --threads and raw Maven -T arguments. if [[ -n "$THREADS" ]]; then HAS_MVN_THREADS=false PREV_MVN_ARG="" for arg in "$@"; do if [[ "$PREV_MVN_ARG" == "-T" ]]; then HAS_MVN_THREADS=true break fi if [[ "$arg" == "-T" ]]; then PREV_MVN_ARG="-T" continue fi if [[ "$arg" =~ ^-T.+ ]]; then HAS_MVN_THREADS=true break fi PREV_MVN_ARG="" done if [[ "$HAS_MVN_THREADS" == true ]]; then echo "ERROR: Do not combine --threads with Maven -T options. Use only one." >&2 exit 1 fi fi # ----------------------------------------------------------------------------- # Section: Argument Validation # Description: # Ensure mandatory options are provided and mutually exclusive rules are respected. # ----------------------------------------------------------------------------- MISSING_REQUIREMENTS=() if [[ "$PRE_PROFILE" == false && "$RELEASE_PROFILE" == false ]]; then MISSING_REQUIREMENTS+=("--pre or --release must be specified") fi if [[ -z "$SPARK_VER" ]]; then MISSING_REQUIREMENTS+=("--sparkver must be specified") fi if [[ -z "$SCALA_VER" ]]; then MISSING_REQUIREMENTS+=("--scalaver must be specified") fi if [[ "${#MISSING_REQUIREMENTS[@]}" -gt 0 ]]; then echo "ERROR: Missing required arguments:" >&2 for req in "${MISSING_REQUIREMENTS[@]}"; do echo " * $req" >&2 done echo echo "Use '$0 --help' for usage information" >&2 exit 1 fi if [[ "$PRE_PROFILE" == true && "$RELEASE_PROFILE" == true ]]; then echo "ERROR: Cannot use both --pre and --release simultaneously" >&2 exit 1 fi # ----------------------------------------------------------------------------- # Section: Build Argument Composition # Description: # Assemble Maven command arguments according to user options. # ----------------------------------------------------------------------------- CLEAN_ARGS=() if [[ "$CLEAN" == true ]]; then CLEAN_ARGS+=("clean") fi BUILD_ARGS=() if [[ "$SKIP_TESTS" == true ]]; then BUILD_ARGS+=("install" "-DskipTests") else BUILD_ARGS+=("install") fi if [[ "$SPARK_TESTS" == true ]]; then BUILD_ARGS+=("-Pspark-tests") fi if [[ "$PRE_PROFILE" == true ]]; then BUILD_ARGS+=("-Ppre") fi if [[ "$RELEASE_PROFILE" == true ]]; then BUILD_ARGS+=("-Prelease") fi if [[ -n "$SPARK_VER" ]]; then BUILD_ARGS+=("-Pspark-$SPARK_VER") fi if [[ -n "$SCALA_VER" ]]; then BUILD_ARGS+=("-Pscala-$SCALA_VER") fi if [[ -n "$CELEBORN_VER" ]]; then BUILD_ARGS+=("-Pceleborn,celeborn-$CELEBORN_VER") fi if [[ -n "$UNIFFLE_VER" ]]; then BUILD_ARGS+=("-Puniffle,uniffle-$UNIFFLE_VER") fi if [[ -n "$PAIMON_VER" ]]; then BUILD_ARGS+=("-Ppaimon,paimon-$PAIMON_VER") fi if [[ -n "$FLINK_VER" ]]; then BUILD_ARGS+=("-Pflink-$FLINK_VER") fi if [[ -n "$ICEBERG_VER" ]]; then BUILD_ARGS+=("-Piceberg-$ICEBERG_VER") fi if [[ -n "$HUDI_VER" ]]; then BUILD_ARGS+=("-Phudi-$HUDI_VER") fi # Configure Maven build threads: # - local builds default to Maven's single-threaded behavior # - docker builds default to -T8 unless overridden if [[ -n "$THREADS" ]]; then BUILD_ARGS+=("-T$THREADS") elif [[ "$USE_DOCKER" == true ]]; then BUILD_ARGS+=("-T8") fi MVN_ARGS=("${CLEAN_ARGS[@]}" "${BUILD_ARGS[@]}") # ----------------------------------------------------------------------------- # Section: Build Info Generation # Description: # Generate auron-build-info.properties for build metadata tracking. # ----------------------------------------------------------------------------- BUILD_INFO_FILE="common/src/main/resources/auron-build-info.properties" mkdir -p "$(dirname "$BUILD_INFO_FILE")" JAVA_VERSION=$(java -version 2>&1 | head -n 1 | awk '{print $3}' | tr -d '"') PROJECT_VERSION=$(./build/mvn help:evaluate -N -Dexpression=project.version -Pspark-${SPARK_VER} -q -DforceStdout 2>/dev/null) RUST_VERSION=$(rustc --version | awk '{print $2}') BUILD_BRANCH="${AURON_BUILD_BRANCH:-$(git rev-parse --abbrev-ref HEAD 2>/dev/null)}" BUILD_REVISION="${AURON_BUILD_REVISION:-$(git rev-parse HEAD 2>/dev/null)}" get_build_info() { case "$1" in "spark.version") echo "${SPARK_VER}" ;; "rust.version") echo "${RUST_VERSION}" ;; "java.version") echo "${JAVA_VERSION}" ;; "project.version") echo "${PROJECT_VERSION}" ;; "scala.version") echo "${SCALA_VER}" ;; "celeborn.version") echo "${CELEBORN_VER}" ;; "uniffle.version") echo "${UNIFFLE_VER}" ;; "paimon.version") echo "${PAIMON_VER}" ;; "flink.version") echo "${FLINK_VER}" ;; "iceberg.version") echo "${ICEBERG_VER}" ;; "hudi.version") echo "${HUDI_VER}" ;; "build.branch") echo "${BUILD_BRANCH}" ;; "build.revision") echo "${BUILD_REVISION}" ;; "build.timestamp") echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" ;; *) echo "" ;; esac } true > "$BUILD_INFO_FILE" for key in \ "spark.version" \ "rust.version" \ "java.version" \ "project.version" \ "scala.version" \ "celeborn.version" \ "uniffle.version" \ "paimon.version" \ "flink.version" \ "iceberg.version" \ "build.branch" \ "build.revision" \ "build.timestamp"; do value="$(get_build_info "$key")" if [[ -n "$value" ]]; then echo "$key=$value" >> "$BUILD_INFO_FILE" fi done # ----------------------------------------------------------------------------- # Section: Build Info Display # Description: # Output generated build info summary to console. # ----------------------------------------------------------------------------- echo "[INFO] Starting Apache Auron build..." if [[ -f "$BUILD_INFO_FILE" ]]; then echo "[INFO] Build configuration (from $BUILD_INFO_FILE):" while IFS='=' read -r key value; do [[ -z "$key" ]] && continue echo "[INFO] $key : $value" done < "$BUILD_INFO_FILE" else echo "[WARN] Build info file not found: $BUILD_INFO_FILE" echo "[INFO] Use '$0 --help' to view usage instructions and supported versions." fi # ----------------------------------------------------------------------------- # Section: Build Execution # Description: # Execute Maven build either locally or inside Docker container. # ----------------------------------------------------------------------------- if [[ "$USE_DOCKER" == true ]]; then echo "[INFO] Compiling inside Docker container using image: $IMAGE_NAME" if [[ "$CLEAN" == true ]]; then # Clean the host-side directory that is mounted into the Docker container. # This avoids "device or resource busy" errors when running `mvn clean` inside the container. echo "[INFO] Docker mode: manually cleaning target-docker contents..." rm -rf ./target-docker/* || echo "[WARN] Failed to clean target-docker/*" fi echo "[INFO] Compiling inside Docker container..." export AURON_BUILD_ARGS="${BUILD_ARGS[*]}" export AURON_BUILD_BRANCH="${BUILD_BRANCH}" export AURON_BUILD_REVISION="${BUILD_REVISION}" export BUILD_CONTEXT="./${IMAGE_NAME}" # Spark 4.x requires JDK 17+, auto-set if not specified if [[ -z "$AURON_JAVA_VERSION" && "$SPARK_VER" == 4.* ]]; then export AURON_JAVA_VERSION="17" fi run_docker_compose_up else echo "[INFO] Compiling locally with maven args: $MVN_CMD ${MVN_ARGS[@]} ${MVN_D_ARGS[*]} $@" "$MVN_CMD" "${MVN_ARGS[@]}" "${MVN_D_ARGS[@]}" "$@" fi