#!/usr/bin/groovy /* Copyright (C) 2019-2021 Intel Corporation * All rights reserved. * * This file is part of the DAOS Project. It is subject to the license terms * in the LICENSE file found in the top-level directory of this distribution * and at https://img.shields.io/badge/License-Apache%202.0-blue.svg. * No part of the DAOS Project, including this file, may be copied, modified, * propagated, or distributed except according to the terms contained in the * LICENSE file. */ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ boolean doc_only_change() { if (cachedCommitPragma(pragma: 'Doc-only') == 'true') { return true } if (cachedCommitPragma(pragma: 'Doc-only') == 'false') { return false } def rc = sh label: "Determine if doc-only change", script: "CHANGE_ID=${env.CHANGE_ID} " + "TARGET_BRANCH=${target_branch} " + 'ci/doc_only_change.sh', returnStatus: true return rc == 1 } boolean release_candidate() { return !sh(label: "Determine if building (a PR of) an RC", script: "git diff-index --name-only HEAD^ | grep -q TAG && " + "grep -i '[0-9]rc[0-9]' TAG", returnStatus: true) } def scons_faults_args() { // The default build will have BUILD_TYPE=dev; fault injection enabled if ((cachedCommitPragma(pragma: 'faults-enabled', def_val: 'true') == 'true') && !release_candidate()) { return "BUILD_TYPE=dev" } else { return "BUILD_TYPE=release" } } def skip_stage(String stage, boolean def_val = false) { String value = 'false' if (def_val) { value = 'true' } return cachedCommitPragma(pragma: 'Skip-' + stage, def_val: value) == 'true' } boolean quickbuild() { return cachedCommitPragma(pragma: 'Quick-build') == 'true' || quick_functional() } def functional_post_always() { return sh(label: "Job Cleanup", script: 'ci/functional/job_cleanup.sh', returnStatus: true) } String get_daos_packages() { Map stage_info = parseStageInfo() return get_daos_packages(stage_info['target']) } String get_daos_packages(String distro) { String pkgs if (env.TEST_RPMS == 'true') { pkgs = "daos{,-{client,tests,server}}" } else { pkgs = "daos{,-client}" } if (distro.startsWith('ubuntu20')) { return pkgs + "=" + daos_packages_version(distro) } return pkgs + "-" + daos_packages_version(distro) } String pr_repos() { Map stage_info = parseStageInfo() return pr_repos(stage_info['target']) } String pr_repos(String distro) { String repos = "" if (distro == 'centos7') { repos = cachedCommitPragma(pragma: 'PR-repos-el7') } else if (distro == 'leap15') { repos = cachedCommitPragma(pragma: 'PR-repos-leap15') } else if (distro.startsWith('ubuntu20')) { repos = cachedCommitPragma(pragma: 'PR-repos-ubuntu20', cache: commit_pragma_cache) } else { error 'pr_repos not implemented for ' + distro } return repos + ' ' + cachedCommitPragma(pragma: 'PR-repos') } String daos_repo() { if (cachedCommitPragma(pragma: 'RPM-test-version') == '') { return "daos@${env.BRANCH_NAME}:${env.BUILD_NUMBER}" } else { return "" } } String hw_distro_target() { if (env.STAGE_NAME.contains('Hardware')) { if (env.STAGE_NAME.contains('Small')) { return hw_distro('small') } if (env.STAGE_NAME.contains('Medium')) { return hw_distro('medium') } if (env.STAGE_NAME.contains('Large')) { return hw_distro('large') } } Map stage_info = parseStageInfo() return stage_info['target'] } String daos_repos() { String target = hw_distro_target() return daos_repos(target) } String daos_repos(String distro) { return pr_repos(distro) + ' ' + daos_repo() } String unit_packages() { Map stage_info = parseStageInfo() boolean need_qb = quickbuild() if (env.STAGE_NAME.contains('Bullseye')) { need_qb = true } if (stage_info['target'] == 'centos7') { String packages = 'gotestsum openmpi3 ' + 'hwloc-devel argobots ' + 'fuse3-libs fuse3 ' + 'boost-devel ' + 'libisa-l-devel libpmem ' + 'libpmemobj protobuf-c ' + 'spdk-devel libfabric-devel ' + 'pmix numactl-devel ' + 'libipmctl-devel ' + 'python36-tabulate numactl ' + 'libyaml-devel ' + 'valgrind-devel patchelf' if (need_qb) { // TODO: these should be gotten from the Requires: of RPM packages += " spdk-tools mercury-2.0.0~rc1" + " boost-devel libisa-l_crypto libfabric-debuginfo" } return packages } else { error 'unit packages not implemented for ' + stage_info['target'] } } commit_pragma_cache = [:] def cachedCommitPragma(Map config) { if (commit_pragma_cache[config['pragma']]) { return commit_pragma_cache[config['pragma']] } commit_pragma_cache[config['pragma']] = commitPragma(config) return commit_pragma_cache[config['pragma']] } String daos_packages_version() { stage_info = parseStageInfo() return daos_packages_version(stage_info['target']) } String daos_packages_version(String distro) { // commit pragma has highest priority // TODO: this should actually be determined from the PR-repos artifacts String version = cachedCommitPragma(pragma: 'RPM-test-version') if (version != "") { String dist = "" if (version.indexOf('-') > -1) { // only tack on the %{dist} if the release was specified if (distro == "centos7") { dist = ".el7" } else if (distro == "leap15") { dist = ".suse.lp152" } } return version + dist } // use the stash after that unstash distro + '-rpm-version' version = readFile(distro + '-rpm-version').trim() if (version != "") { return version } error "Don't know how to determine package version for " + distro } boolean parallel_build() { // defaults to false // true if Quick-build: true unless Parallel-build: false def pb = cachedCommitPragma(pragma: 'Parallel-build') if (pb == "true" || (quickbuild() && pb != "false")) { return true } return false } String hw_distro(String size) { // Possible values: //'leap15 //'centos7 return cachedCommitPragma(pragma: 'Func-hw-test-' + size + '-distro', def_val: cachedCommitPragma(pragma: 'Func-hw-test-distro', def_val: 'centos7')) } String functional_packages() { String target = hw_distro_target() return functional_packages(target) } String functional_packages(String distro) { String daos_pkgs = get_daos_packages(distro) String pkgs = " openmpi3 hwloc ndctl fio " + "patchutils ior-hpc-daos-1 " + "romio-tests-daos-1 " + "testmpio " + "mpi4py-tests " + "hdf5-mpich-tests " + "hdf5-openmpi3-tests " + "hdf5-vol-daos-mpich2-tests-daos-1 " + "hdf5-vol-daos-openmpi3-tests-daos-1 " + "MACSio-mpich " + "MACSio-openmpi3 " + "mpifileutils-mpich-daos-1 " if (distro == "leap15") { return daos_pkgs + pkgs } else if (distro == "centos7") { // need to exclude openmpi until we remove it from the repo return "--exclude openmpi " + daos_pkgs + pkgs } else if (distro.startsWith('ubuntu20')) { return daos_pkgs + " openmpi-bin ndctl fio" } else { error 'functional_packages not implemented for ' + stage_info['target'] } } // Don't define this as a type or it loses it's global scope target_branch = env.CHANGE_TARGET ? env.CHANGE_TARGET : env.BRANCH_NAME def sanitized_JOB_NAME = JOB_NAME.toLowerCase().replaceAll('/', '-').replaceAll('%2f', '-') // bail out of branch builds that are not on a whitelist if (!env.CHANGE_ID && (!env.BRANCH_NAME.startsWith("weekly-testing") && !env.BRANCH_NAME.startsWith("release/") && env.BRANCH_NAME != "master")) { currentBuild.result = 'SUCCESS' return } // The docker agent setup and the provisionNodes step need to know the // UID that the build agent is running under. cached_uid = 0 def getuid() { if (cached_uid == 0) cached_uid = sh(label: 'getuid()', script: "id -u", returnStdout: true).trim() return cached_uid } // Default priority is 3, lower is better. // The parameter for a job is set using the script/Jenkinsfile from the // previous build of that job, so the first build of any PR will always // run at default because Jenkins sees it as a new job, but subsequent // ones will use the value from here. // The advantage therefore is not to change the priority of PRs, but to // change the master branch itself to run at lower priority, resulting // in faster time-to-result for PRs. String get_priority() { if (env.BRANCH_NAME == 'master') { string p = '2' } else { string p = '' } echo "Build priority set to " + p == '' ? 'default' : p return p } String rpm_test_version() { return cachedCommitPragma(pragma: 'RPM-test-version') } boolean skip_prebuild() { return target_branch == 'weekly-testing' } boolean skip_checkpatch() { return skip_stage('checkpatch') || doc_only_change() || quick_functional() } boolean skip_build() { // always build branch landings as we depend on lastSuccessfulBuild // always having RPMs in it return (env.BRANCH_NAME != target_branch) && skip_stage('build') || doc_only_change() || rpm_test_version() != '' } boolean quick_functional() { return cachedCommitPragma(pragma: 'Quick-Functional') == 'true' } boolean skip_build_rpm(String distro) { return target_branch == 'weekly-testing' || skip_stage('build-' + distro + '-rpm') || distro == 'ubuntu20' && quick_functional() } boolean skip_build_on_centos7_gcc() { return skip_stage('build-centos7-gcc') || quick_functional() } boolean skip_ftest(String distro) { return distro == 'ubuntu20' || skip_stage('func-test') || skip_stage('func-test-vm') || ! tests_in_stage('vm') || skip_stage('func-test-' + distro) } boolean skip_test_rpms_centos7() { return target_branch == 'weekly-testing' || skip_stage('test') || skip_stage('test-centos-rpms') || quick_functional() } boolean skip_scan_rpms_centos7() { return target_branch == 'weekly-testing' || skip_stage('scan-centos-rpms') || quick_functional() } boolean tests_in_stage(String size) { String tags = cachedCommitPragma(pragma: 'Test-tag', def_val: 'pr') def newtags = [] if (size == "vm") { for (String tag in tags.split(" ")) { newtags.add(tag + ",-hw") } tags += ",-hw" } else { if (size == "medium") { size += ",ib2" } for (String tag in tags.split(" ")) { newtags.add(tag + ",hw," + size) } } tags = newtags.join(" ") return sh(label: "Get test list for ${size}", script: """cd src/tests/ftest ./launch.py --list ${tags}""", returnStatus: true) == 0 } boolean skip_ftest_hw(String size) { return env.DAOS_STACK_CI_HARDWARE_SKIP == 'true' || skip_stage('func-test') || skip_stage('func-hw-test') || skip_stage('func-hw-test-' + size) || ! tests_in_stage(size) || (env.BRANCH_NAME == 'master' && ! startedByTimer()) } boolean skip_bandit_check() { return cachedCommitPragma(pragma: 'Skip-python-bandit', def_val: 'true') == 'true' || quick_functional() } boolean skip_build_on_centos7_bullseye() { return env.NO_CI_TESTING == 'true' || skip_stage('bullseye', true) || quick_functional() } boolean skip_build_on_centos7_gcc_debug() { return skip_stage('build-centos7-gcc-debug') || quickbuild() } boolean skip_build_on_centos7_gcc_release() { return skip_stage('build-centos7-gcc-release') || quickbuild() } boolean skip_build_on_landing_branch() { return env.BRANCH_NAME != target_branch || quickbuild() } boolean skip_build_on_ubuntu_clang() { return target_branch == 'weekly-testing' || skip_stage('build-ubuntu-clang') || quickbuild() } boolean skip_build_on_leap15_gcc() { return skip_stage('build-leap15-gcc') || quickbuild() } boolean skip_build_on_leap15_icc() { return target_branch == 'weekly-testing' || skip_stage('build-leap15-icc') || quickbuild() } boolean skip_unit_testing_stage() { return env.NO_CI_TESTING == 'true' || (skip_stage('build') && rpm_test_version() == '') || doc_only_change() || skip_build_on_centos7_gcc() || skip_stage('unit-tests') } boolean skip_coverity() { return skip_stage('coverity-test') || quick_functional() } boolean skip_testing_stage() { return env.NO_CI_TESTING == 'true' || (skip_stage('build') && rpm_test_version() == '') || doc_only_change() || skip_stage('test') } boolean skip_unit_test() { return skip_stage('unit-test') || skip_stage('run_test') } boolean skip_bullseye_report() { return env.BULLSEYE == null || skip_stage('bullseye', true) } String quick_build_deps(String distro) { String rpmspec_args = "" if (distro == "leap15") { rpmspec_args = "--define dist\\ .suse.lp152 " + "--undefine rhel " + "--define suse_version\\ 1502" } else if (distro == "centos7") { rpmspec_args = "--undefine suse_version " + "--define rhel\\ 7" } else { error("Unknown distro: ${distro} in quick_build_deps()") } return sh(label: 'Get Quickbuild dependencies', script: "rpmspec -q " + "--srpm " + rpmspec_args + ' ' + "--requires utils/rpms/daos.spec " + "2>/dev/null", returnStdout: true) } pipeline { agent { label 'lightweight' } triggers { cron(env.BRANCH_NAME == 'master' ? 'TZ=America/Toronto\n0 0,12 * * *\n' : '' + env.BRANCH_NAME == 'weekly-testing' ? 'H 0 * * 6' : '') } environment { BULLSEYE = credentials('bullseye_license_key') GITHUB_USER = credentials('daos-jenkins-review-posting') SSH_KEY_ARGS = "-ici_key" CLUSH_ARGS = "-o$SSH_KEY_ARGS" TEST_RPMS = cachedCommitPragma(pragma: 'RPM-test', def_val: 'true') SCONS_FAULTS_ARGS = scons_faults_args() } options { // preserve stashes so that jobs can be started at the test stage preserveStashes(buildCount: 5) ansiColor('xterm') buildDiscarder(logRotator(artifactDaysToKeepStr: '100')) } parameters { string(name: 'BuildPriority', defaultValue: get_priority(), description: 'Priority of this build. DO NOT USE WITHOUT PERMISSION.') } stages { stage('Cancel Previous Builds') { when { changeRequest() } steps { cancelPreviousBuilds() } } stage('Pre-build') { when { beforeAgent true expression { ! skip_prebuild() } } parallel { stage('checkpatch') { when { beforeAgent true expression { ! skip_checkpatch() } } agent { dockerfile { filename 'Dockerfile.centos.7' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs() + " -t ${sanitized_JOB_NAME}-centos7 " } } steps { checkPatch user: GITHUB_USER_USR, password: GITHUB_USER_PSW, ignored_files: "src/control/vendor/*:" + "src/include/daos/*.pb-c.h:" + "src/common/*.pb-c.[ch]:" + "src/mgmt/*.pb-c.[ch]:" + "src/iosrv/*.pb-c.[ch]:" + "src/security/*.pb-c.[ch]:" + "src/client/java/daos-java/src/main/java/io/daos/dfs/uns/*:" + "src/client/java/daos-java/src/main/java/io/daos/obj/attr/*:" + "src/client/java/daos-java/src/main/native/include/daos_jni_common.h:" + "src/client/java/daos-java/src/main/native/*.pb-c.[ch]:" + "src/client/java/daos-java/src/main/native/include/*.pb-c.[ch]:" + "*.crt:" + "*.pem:" + "*_test.go:" + "src/cart/_structures_from_macros_.h:" + "src/tests/ftest/*.patch:" + "src/tests/ftest/large_stdout.txt" } post { always { archiveArtifacts artifacts: 'pylint.log', allowEmptyArchive: true /* when JENKINS-39203 is resolved, can probably use stepResult here and remove the remaining post conditions stepResult name: env.STAGE_NAME, context: 'build/' + env.STAGE_NAME, result: ${currentBuild.currentResult} */ } /* temporarily moved into stepResult due to JENKINS-39203 success { githubNotify credentialsId: 'daos-jenkins-commit-status', description: env.STAGE_NAME, context: 'pre-build/' + env.STAGE_NAME, status: 'SUCCESS' } unstable { githubNotify credentialsId: 'daos-jenkins-commit-status', description: env.STAGE_NAME, context: 'pre-build/' + env.STAGE_NAME, status: 'FAILURE' } failure { githubNotify credentialsId: 'daos-jenkins-commit-status', description: env.STAGE_NAME, context: 'pre-build/' + env.STAGE_NAME, status: 'ERROR' } */ } } // stage('checkpatch') stage('Python Bandit check') { when { beforeAgent true expression { ! skip_bandit_check() } } agent { dockerfile { filename 'Dockerfile.code_scanning' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs() } } steps { pythonBanditCheck() } post { always { // Bandit will have empty results if it does not // find any issues. junit testResults: 'bandit.xml', allowEmptyResults: true } } } // stage('Python Bandit check') } } stage('Build') { /* Don't use failFast here as whilst it avoids using extra resources * and gives faster results for PRs it's also on for master where we * do want complete results in the case of partial failure */ //failFast true when { beforeAgent true expression { ! skip_build() } } parallel { stage('Build RPM on CentOS 7') { agent { dockerfile { filename 'Dockerfile.mockbuild' dir 'utils/rpms/packaging' label 'docker_runner' additionalBuildArgs dockerBuildArgs() args '--group-add mock --cap-add=SYS_ADMIN --privileged=true' } } steps { buildRpm() } post { success { buildRpmPost condition: 'success' } unstable { buildRpmPost condition: 'unstable' } failure { buildRpmPost condition: 'failure' } unsuccessful { buildRpmPost condition: 'unsuccessful' } cleanup { buildRpmPost condition: 'cleanup' } } } stage('Build RPM on Leap 15') { when { beforeAgent true expression { ! skip_build_rpm('leap15') } } agent { dockerfile { filename 'Dockerfile.mockbuild' dir 'utils/rpms/packaging' label 'docker_runner' additionalBuildArgs dockerBuildArgs() args '--group-add mock --cap-add=SYS_ADMIN --privileged=true' } } steps { buildRpm() } post { success { buildRpmPost condition: 'success' } unstable { buildRpmPost condition: 'unstable' } failure { buildRpmPost condition: 'failure' } unsuccessful { buildRpmPost condition: 'unsuccessful' } cleanup { buildRpmPost condition: 'cleanup' } } } stage('Build DEB on Ubuntu 20.04') { when { beforeAgent true expression { ! skip_build_rpm('ubuntu20') } } agent { dockerfile { filename 'Dockerfile.ubuntu.20.04' dir 'utils/rpms/packaging' label 'docker_runner' args '--privileged=true' additionalBuildArgs dockerBuildArgs() args '--cap-add=SYS_ADMIN --privileged=true' } } steps { buildRpm() } post { success { buildRpmPost condition: 'success' } unstable { buildRpmPost condition: 'unstable' } failure { buildRpmPost condition: 'failure' } unsuccessful { buildRpmPost condition: 'unsuccessful' } cleanup { buildRpmPost condition: 'cleanup' } } } stage('Build on CentOS 7') { when { beforeAgent true expression { ! skip_build_on_centos7_gcc() } } agent { dockerfile { filename 'Dockerfile.centos.7' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg QUICKBUILD_DEPS="' + quick_build_deps('centos7') + '"' + ' --build-arg REPOS="' + pr_repos() + '"' } } steps { sconsBuild parallel_build: parallel_build(), stash_files: 'ci/test_files_to_stash.txt', scons_args: scons_faults_args() } post { always { recordIssues enabledForFailure: true, aggregatingResults: true, id: "analysis-gcc-centos7", tools: [ gcc4(pattern: 'centos7-gcc-build.log'), cppCheck(pattern: 'centos7-gcc-build.log') ], filters: [ excludeFile('.*\\/_build\\.external\\/.*'), excludeFile('_build\\.external\\/.*') ] } success { sh "rm -rf _build.external" } unsuccessful { sh """if [ -f config.log ]; then mv config.log config.log-centos7-gcc fi""" archiveArtifacts artifacts: 'config.log-centos7-gcc', allowEmptyArchive: true } } } stage('Build on CentOS 7 Bullseye') { when { beforeAgent true expression { ! skip_build_on_centos7_bullseye() } } agent { dockerfile { filename 'Dockerfile.centos.7' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg BULLSEYE=' + env.BULLSEYE + ' --build-arg QUICKBUILD_DEPS="' + quick_build_deps('centos7') + '"' + ' --build-arg REPOS="' + pr_repos() + '"' } } steps { sconsBuild parallel_build: parallel_build(), stash_files: 'ci/test_files_to_stash.txt', scons_args: scons_faults_args() } post { always { recordIssues enabledForFailure: true, aggregatingResults: true, id: "analysis-covc-centos7", tools: [ gcc4(pattern: 'centos7-covc-build.log'), cppCheck(pattern: 'centos7-covc-build.log') ], filters: [ excludeFile('.*\\/_build\\.external\\/.*'), excludeFile('_build\\.external\\/.*') ] } success { sh "rm -rf _build.external" } unsuccessful { sh """if [ -f config.log ]; then mv config.log config.log-centos7-covc fi""" archiveArtifacts artifacts: 'config.log-centos7-covc', allowEmptyArchive: true } } } stage('Build on CentOS 7 debug') { when { beforeAgent true expression { ! skip_build_on_centos7_gcc_debug() } } agent { dockerfile { filename 'Dockerfile.centos.7' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg QUICKBUILD_DEPS="' + quick_build_deps('centos7') + '"' + ' --build-arg REPOS="' + pr_repos() + '"' } } steps { sconsBuild parallel_build: parallel_build() } post { always { recordIssues enabledForFailure: true, aggregatingResults: true, id: "analysis-gcc-centos7-debug", tools: [ gcc4(pattern: 'centos7-gcc-debug-build.log'), cppCheck(pattern: 'centos7-gcc-debug-build.log') ], filters: [ excludeFile('.*\\/_build\\.external\\/.*'), excludeFile('_build\\.external\\/.*') ] } success { sh "rm -rf _build.external" } unsuccessful { sh """if [ -f config.log ]; then mv config.log config.log-centos7-gcc-debug fi""" archiveArtifacts artifacts: 'config.log-centos7-gcc-debug', allowEmptyArchive: true } } } stage('Build on CentOS 7 release') { when { beforeAgent true expression { ! skip_build_on_centos7_gcc_release() } } agent { dockerfile { filename 'Dockerfile.centos.7' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg QUICKBUILD_DEPS="' + quick_build_deps('centos7') + '"' + ' --build-arg REPOS="' + pr_repos() + '"' } } steps { sconsBuild parallel_build: parallel_build() } post { always { recordIssues enabledForFailure: true, aggregatingResults: true, id: "analysis-gcc-centos7-release", tools: [ gcc4(pattern: 'centos7-gcc-release-build.log'), cppCheck(pattern: 'centos7-gcc-release-build.log') ], filters: [excludeFile('.*\\/_build\\.external\\/.*'), excludeFile('_build\\.external\\/.*')] } success { sh "rm -rf _build.external" } unsuccessful { sh """if [ -f config.log ]; then mv config.log config.log-centos7-gcc-release fi""" archiveArtifacts artifacts: 'config.log-centos7-gcc-release', allowEmptyArchive: true } } } stage('Build on CentOS 7 with Clang') { when { beforeAgent true expression { ! skip_build_on_landing_branch() } } agent { dockerfile { filename 'Dockerfile.centos.7' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg QUICKBUILD_DEPS="' + quick_build_deps('centos7') + '"' } } steps { sconsBuild parallel_build: parallel_build(), scons_args: scons_faults_args() } post { always { recordIssues enabledForFailure: true, aggregatingResults: true, id: "analysis-centos7-clang", tools: [ clang(pattern: 'centos7-clang-build.log'), cppCheck(pattern: 'centos7-clang-build.log') ], filters: [ excludeFile('.*\\/_build\\.external\\/.*'), excludeFile('_build\\.external\\/.*') ] } success { sh "rm -rf _build.external" } unsuccessful { sh """if [ -f config.log ]; then mv config.log config.log-centos7-clang fi""" archiveArtifacts artifacts: 'config.log-centos7-clang', allowEmptyArchive: true } } } stage('Build on Ubuntu 20.04') { when { beforeAgent true expression { ! skip_build_on_landing_branch() } } agent { dockerfile { filename 'Dockerfile.ubuntu.20.04' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs() + " -t ${sanitized_JOB_NAME}-ubuntu20.04" } } steps { sconsBuild parallel_build: parallel_build(), scons_args: scons_faults_args() } post { always { recordIssues enabledForFailure: true, aggregatingResults: true, id: "analysis-ubuntu20", tools: [ gcc4(pattern: 'ubuntu20.04-gcc-build.log'), cppCheck(pattern: 'ubuntu20.04-gcc-build.log') ], filters: [ excludeFile('.*\\/_build\\.external\\/.*'), excludeFile('_build\\.external\\/.*') ] } success { sh "rm -rf _build.external" } unsuccessful { sh """if [ -f config.log ]; then mv config.log config.log-ubuntu20.04-gcc fi""" archiveArtifacts artifacts: 'config.log-ubuntu20.04-gcc', allowEmptyArchive: true } } } stage('Build on Ubuntu 20.04 with Clang') { when { beforeAgent true expression { ! skip_build_on_ubuntu_clang() } } agent { dockerfile { filename 'Dockerfile.ubuntu.20.04' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs() + " -t ${sanitized_JOB_NAME}-ubuntu20.04" } } steps { sconsBuild parallel_build: parallel_build(), scons_args: scons_faults_args() } post { always { recordIssues enabledForFailure: true, aggregatingResults: true, id: "analysis-ubuntu20-clang", tools: [ clang(pattern: 'ubuntu20.04-clang-build.log'), cppCheck(pattern: 'ubuntu20.04-clang-build.log') ], filters: [ excludeFile('.*\\/_build\\.external\\/.*'), excludeFile('_build\\.external\\/.*') ] } success { sh "rm -rf _build.external" } unsuccessful { sh """if [ -f config.log ]; then mv config.log config.log-ubuntu20.04-clang fi""" archiveArtifacts artifacts: 'config.log-ubuntu20.04-clang', allowEmptyArchive: true } } } stage('Build on Leap 15') { when { beforeAgent true expression { ! skip_build_on_leap15_gcc() } } agent { dockerfile { filename 'Dockerfile.leap.15' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + " -t ${sanitized_JOB_NAME}-leap15 " + ' --build-arg QUICKBUILD_DEPS="' + quick_build_deps('leap15') + '"' + ' --build-arg REPOS="' + pr_repos() + '"' } } steps { sconsBuild parallel_build: parallel_build(), stash_files: 'ci/test_files_to_stash.txt', scons_args: scons_faults_args() } post { always { recordIssues enabledForFailure: true, aggregatingResults: true, id: "analysis-gcc-leap15", tools: [ gcc4(pattern: 'leap15-gcc-build.log'), cppCheck(pattern: 'leap15-gcc-build.log') ], filters: [ excludeFile('.*\\/_build\\.external\\/.*'), excludeFile('_build\\.external\\/.*') ] } success { sh "rm -rf _build.external" } unsuccessful { sh """if [ -f config.log ]; then mv config.log config.log-leap15-gcc fi""" archiveArtifacts artifacts: 'config.log-leap15-gcc', allowEmptyArchive: true } } } stage('Build on Leap 15 with Clang') { when { beforeAgent true expression { ! skip_build_on_landing_branch() } } agent { dockerfile { filename 'Dockerfile.leap.15' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs() + " -t ${sanitized_JOB_NAME}-leap15" } } steps { sconsBuild parallel_build: parallel_build(), scons_args: scons_faults_args() } post { always { recordIssues enabledForFailure: true, aggregatingResults: true, id: "analysis-leap15-clang", tools: [ clang(pattern: 'leap15-clang-build.log'), cppCheck(pattern: 'leap15-clang-build.log') ], filters: [ excludeFile('.*\\/_build\\.external\\/.*'), excludeFile('_build\\.external\\/.*') ] } success { sh "rm -rf _build.external" } unsuccessful { sh """if [ -f config.log ]; then mv config.log config.log-leap15-clang fi""" archiveArtifacts artifacts: 'config.log-leap15-clang', allowEmptyArchive: true } } } stage('Build on Leap 15 with Intel-C and TARGET_PREFIX') { when { beforeAgent true expression { ! skip_build_on_leap15_icc() } } agent { dockerfile { filename 'Dockerfile.leap.15' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs() + " -t ${sanitized_JOB_NAME}-leap15" args '-v /opt/intel:/opt/intel' } } steps { sconsBuild parallel_build: parallel_build(), scons_args: scons_faults_args() } post { always { recordIssues enabledForFailure: true, aggregatingResults: true, id: "analysis-leap15-intelc", tools: [ intel(pattern: 'leap15-icc-build.log'), cppCheck(pattern: 'leap15-icc-build.log') ], filters: [ excludeFile('.*\\/_build\\.external\\/.*'), excludeFile('_build\\.external\\/.*') ] } success { sh "rm -rf _build.external" } unsuccessful { sh """if [ -f config.log ]; then mv config.log config.log-leap15-intelc fi""" archiveArtifacts artifacts: 'config.log-leap15-intelc', allowEmptyArchive: true } } } } } stage('Unit Tests') { when { beforeAgent true expression { ! skip_unit_testing_stage() } } parallel { stage('Unit Test') { when { beforeAgent true expression { ! skip_unit_test() } } agent { label 'ci_vm1' } steps { unitTest timeout_time: 30, inst_repos: pr_repos(), inst_rpms: unit_packages() } post { always { unitTestPost artifacts: ['unit_test_logs/*'] } } } stage('NLT') { when { beforeAgent true expression { ! skip_stage('nlt') } } agent { label 'ci_nlt_1' } steps { unitTest timeout_time: 20, inst_repos: pr_repos(), test_script: 'ci/unit/test_nlt.sh', inst_rpms: unit_packages() } post { always { unitTestPost artifacts: ['nlt_logs/*'], testResults: 'None', always_script: 'ci/unit/test_nlt_post.sh', valgrind_stash: 'centos7-gcc-nlt-memcheck' } } } stage('Unit Test Bullseye') { when { beforeAgent true expression { ! skip_stage('bullseye', true) } } agent { label 'ci_vm1' } steps { unitTest timeout_time: 60, ignore_failure: true, inst_repos: pr_repos(), inst_rpms: unit_packages() } post { always { // This is only set while dealing with issues // caused by code coverage instrumentation affecting // test results, and while code coverage is being // added. unitTestPost ignore_failure: true, artifacts: ['covc_test_logs/*', 'covc_vm_test/**'] } } } // stage('Unit test Bullseye') stage('Unit Test with memcheck') { when { beforeAgent true expression { ! skip_stage('unit-test-memcheck') } } agent { label 'ci_vm1' } steps { unitTest timeout_time: 30, ignore_failure: true, inst_repos: pr_repos(), inst_rpms: unit_packages() } post { always { unitTestPost artifacts: ['unit_test_memcheck_logs.tar.gz', 'unit_test_memcheck_logs/*.log'], valgrind_stash: 'centos7-gcc-unit-memcheck' } } } // stage('Unit Test with memcheck') } } stage('Test') { when { beforeAgent true expression { ! skip_testing_stage() } } parallel { stage('Coverity on CentOS 7') { when { beforeAgent true expression { ! skip_coverity() } } agent { dockerfile { filename 'Dockerfile.centos.7' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs(qb: true) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg QUICKBUILD_DEPS="' + quick_build_deps('centos7') + '"' + ' --build-arg REPOS="' + pr_repos() + '"' } } steps { sconsBuild coverity: "daos-stack/daos", parallel_build: parallel_build() } post { success { coverityPost condition: 'success' } unsuccessful { coverityPost condition: 'unsuccessful' } } } // stage('Coverity on CentOS 7') stage('Functional on CentOS 7') { when { beforeAgent true expression { ! skip_ftest('el7') } } agent { label 'ci_vm9' } steps { functionalTest inst_repos: daos_repos(), inst_rpms: functional_packages() } post { always { functionalTestPost() } } } // stage('Functional on CentOS 7') stage('Functional on Leap 15') { when { beforeAgent true expression { ! skip_ftest('leap15') } } agent { label 'ci_vm9' } steps { functionalTest inst_repos: daos_repos(), inst_rpms: functional_packages() } post { always { functionalTestPost() } } // post } // stage('Functional on Leap 15') stage('Functional on Ubuntu 20.04') { when { beforeAgent true expression { ! skip_ftest('ubuntu20') } } agent { label 'ci_vm9' } steps { functionalTest inst_repos: daos_repos(), inst_rpms: functional_packages() } post { always { functionalTestPost() } } // post } // stage('Functional on Ubuntu 20.04') stage('Functional_Hardware_Small') { when { beforeAgent true expression { ! skip_ftest_hw('small') } } agent { // 2 node cluster with 1 IB/node + 1 test control node label 'ci_nvme3' } steps { functionalTest target: hw_distro_target(), inst_repos: daos_repos(), inst_rpms: functional_packages() } post { always { functionalTestPost() } } } // stage('Functional_Hardware_Small') stage('Functional_Hardware_Medium') { when { beforeAgent true expression { ! skip_ftest_hw('medium') } } agent { // 4 node cluster with 2 IB/node + 1 test control node label 'ci_nvme5' } steps { functionalTest target: hw_distro_target(), inst_repos: daos_repos(), inst_rpms: functional_packages() } post { always { functionalTestPost() } } } // stage('Functional_Hardware_Medium') stage('Functional_Hardware_Large') { when { beforeAgent true expression { ! skip_ftest_hw('large') } } agent { // 8+ node cluster with 1 IB/node + 1 test control node label 'ci_nvme9' } steps { functionalTest target: hw_distro_target(), inst_repos: daos_repos(), inst_rpms: functional_packages() } post { always { functionalTestPost() } } } // stage('Functional_Hardware_Large') stage('Test CentOS 7 RPMs') { when { beforeAgent true expression { ! skip_test_rpms_centos7() } } agent { label 'ci_vm1' } steps { testRpm inst_repos: daos_repos(), daos_pkg_version: daos_packages_version() } } // stage('Test CentOS 7 RPMs') stage('Scan CentOS 7 RPMs') { when { beforeAgent true expression { ! skip_scan_rpms_centos7() } } agent { label 'ci_vm1' } steps { scanRpms inst_repos: daos_repos(), daos_pkg_version: daos_packages_version(), inst_rpms: 'clamav clamav-devel', test_script: 'ci/rpm/scan_daos.sh', junit_files: 'maldetect.xml' } post { always { junit 'maldetect.xml' } } } // stage('Scan CentOS 7 RPMs') } // parallel } // stage('Test') stage ('Test Report') { parallel { stage('Bullseye Report') { when { beforeAgent true expression { ! skip_bullseye_report() } } agent { dockerfile { filename 'Dockerfile.centos.7' dir 'utils/docker' label 'docker_runner' additionalBuildArgs dockerBuildArgs(qb: quickbuild()) + " -t ${sanitized_JOB_NAME}-centos7 " + ' --build-arg BULLSEYE=' + env.BULLSEYE + ' --build-arg QUICKBUILD_DEPS="' + quick_build_deps('centos7') + '"' + ' --build-arg REPOS="' + pr_repos() + '"' } } steps { // The coverage_healthy is primarily set here // while the code coverage feature is being implemented. cloverReportPublish coverage_stashes: ['centos7-covc-unit-cov'], coverage_healthy: [methodCoverage: 0, conditionalCoverage: 0, statementCoverage: 0], ignore_failure: true } } // stage('Bullseye Report') } // parallel } // stage ('Test Report') } // stages post { always { valgrindReportPublish valgrind_stashes: ['centos7-gcc-nlt-memcheck', 'centos7-gcc-unit-memcheck'] } unsuccessful { notifyBrokenBranch branches: target_branch } } // post }