# Common (multistage) args ARG D_OS="ubuntu22.04" ARG D_ARCH="x86_64" ARG D_CONTAINER_VER="0" ARG D_DOCA_VERSION="2.9.1" ARG D_OFED_VERSION="24.10-1.1.4.0" ARG D_KERNEL_VER="5.15.0-25-generic" ARG D_OFED_SRC_DOWNLOAD_PATH="/run/mellanox/src" ARG OFED_SRC_LOCAL_DIR=${D_OFED_SRC_DOWNLOAD_PATH}/MLNX_OFED_SRC-${D_OFED_VERSION} ARG STIG_COMPLIANT=false ARG D_ENABLE_DKMS=false # Common for build and final clean image of precompiled driver container ARG D_BASE_IMAGE="ubuntu:22.04" ################################################################## # Stage: build go binary for entrypoint FROM golang:1.26 AS go_builder # Set GOPROXY if provided ARG GOPROXY ENV GOPROXY=$GOPROXY WORKDIR /workspace COPY entrypoint/go.mod go.mod COPY entrypoint/go.sum go.sum RUN go mod download COPY entrypoint/ . RUN TARGETARCH=${D_ARCH} TARGETOS=linux make build ################################################################## # Stage: Minimal base image update and install common requirements FROM $D_BASE_IMAGE AS base ARG D_APT_REMOVE="" ARG D_OFED_VERSION ARG D_CONTAINER_VER ARG D_OFED_SRC_DOWNLOAD_PATH ARG STIG_COMPLIANT ENV NVIDIA_NIC_DRIVER_VER=${D_OFED_VERSION} ENV NVIDIA_NIC_CONTAINER_VER=${D_CONTAINER_VER} WORKDIR /root RUN set -x && \ for source in ${D_APT_REMOVE}; do rm -f /etc/apt/sources.list.d/${source}.list; done && \ # Perform distro update and install prerequirements apt-get -yq update && \ DEBIAN_FRONTEND=noninteractive apt-get -yq -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade && \ DEBIAN_FRONTEND=noninteractive apt-get -yq -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install apt-utils \ # Driver build / install script requirements perl pciutils kmod lsof python3 dh-python \ # Container functional requirements jq iproute2 udev ethtool ca-certificates \ # NFS userspace tools (mount.nfs) — copied to host at runtime when ENABLE_NFSRDMA=true nfs-common COPY --from=go_builder /workspace/build/entrypoint /root/entrypoint ADD ./entrypoint.sh /root/entrypoint.sh ADD ./loader.sh /root/loader.sh # dockerfile COPY instruction limitation requires adding a file - even if it's an empty/dummy file COPY stig-fixer.sh /tmp/stig-fixer.sh # run STIG fixer script for STIG complaint image builds RUN set -x && \ if [ "$STIG_COMPLIANT" = "true" ]; then \ chmod +x /tmp/stig-fixer.sh && \ /tmp/stig-fixer.sh ; \ fi && \ rm -f /tmp/stig-fixer.sh ENTRYPOINT ["/root/loader.sh"] ############################################################################################## # Stage: Download NVIDIA driver sources and install src driver container packages requirements FROM base AS driver-src # Inherited global args ARG D_OS ARG D_DOCA_VERSION ARG D_OFED_VERSION ARG D_OFED_SRC_DOWNLOAD_PATH # Stage args ARG D_OFED_BASE_URL="https://linux.mellanox.com/public/repo/doca/${D_DOCA_VERSION}/SOURCES/mlnx_ofed" ARG D_OFED_SRC_TYPE="debian-" # Up to DOCA 3.5 the sources ship as MLNX_OFED_SRC-debian-.tgz, which bundles # install.pl. From DOCA 3.6 they are published at # https://linux.mellanox.com/public/repo/doca//SOURCES/mlnx_ofed/ # as DOCA_BASE_SRC-debian--.tar.gz # (for example DOCA_BASE_SRC-debian-3.6.0-069000.tar.gz) and contain no installer. # is the numeric suffix of D_OFED_VERSION (26.10-069000 -> 069000), or the # suffix already present on D_DOCA_VERSION (3.6.0-069000). The fetch below builds that # name unless D_OFED_URL_PATH is a local archive or a full URL. # Both layouts are detected and handled below. ARG D_OFED_SRC_ARCHIVE="MLNX_OFED_SRC-${D_OFED_SRC_TYPE}${D_OFED_VERSION}.tgz" ARG D_OFED_URL_DEFAULT="${D_OFED_BASE_URL}/${D_OFED_SRC_ARCHIVE}" # Local context file or full URL. Jenkins and Kaniko pass the archive copied into # the build context (ofed_src.tgz, DOCA_BASE_SRC-....tar.gz). Left unset, ADD # copies README.md and the fetch below replaces it with the public archive. ARG D_OFED_URL_PATH="README.md" ENV NVIDIA_NIC_DRIVER_PATH="${D_OFED_SRC_DOWNLOAD_PATH}/MLNX_OFED_SRC-${D_OFED_VERSION}" WORKDIR /root RUN set -x && \ apt-get update RUN set -x && \ # Ubuntu 22.04 and 24.04 kernels were built with GCC 12, while build-essential installs a newer # default gcc, so those releases must build modules with gcc-12. Ubuntu 20.04 already matches. # Ubuntu 26.04 kernels (7.0) were built with GCC 15; forcing gcc-12 there fails the # mlnx-ofed-kernel package build. \ if echo "$D_OS" | grep -qE '^ubuntu(22|24)\.'; then \ GCC_VER="-12"; \ fi && \ # Install prerequirements \ DEBIAN_FRONTEND=noninteractive apt-get -yq -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install curl \ dkms make autoconf autotools-dev chrpath automake hostname debhelper dh-exec gcc$GCC_VER quilt libc6-dev build-essential pkg-config ubuntu-advantage-tools \ bison flex libelf-dev zstd && \ # On Ubuntu 24.04+ dkms is >= 3.0; the dh_dkms helper moved out of the dkms package into a # standalone dh-dkms package. mlnx-ofed-kernel's Build-Depends is "dh-dkms | dkms (<< 3.0)", # so dpkg-buildpackage fails on Noble+ unless dh-dkms is installed. Older Ubuntu releases # don't ship dh-dkms (and don't need it), so guard the install on the distro. \ if echo "$D_OS" | grep -qE "ubuntu(2[4-9]|[3-9][0-9])"; then \ DEBIAN_FRONTEND=noninteractive apt-get -yq -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install dh-dkms; \ fi && \ # Cleanup \ apt-get clean autoclean && apt install --reinstall -y ca-certificates && \ rm -rf /var/lib/apt/lists/* && update-ca-certificates # 'build-essential' installs `gcc`. On 22.04 and 24.04 that default is newer than the GCC 12 # the kernel was built with, so point gcc at gcc-12. 20.04 and 26.04+ keep the distro compiler. RUN if echo "$D_OS" | grep -qE '^ubuntu(22|24)\.'; then ln -fs gcc-12 /usr/bin/gcc; fi # Download NVIDIA NIC driver RUN mkdir -p ${D_OFED_SRC_DOWNLOAD_PATH} WORKDIR ${D_OFED_SRC_DOWNLOAD_PATH} # ADD is what Kaniko, docker, and podman already use. A context file is unpacked # when it is a tar; a full URL is stored as a file. The default source is # README.md, plain text so ADD does not unpack it. ADD cannot pick the archive # name from the DOCA version, so the next step downloads the public archive # when that default is still the source. ADD ${D_OFED_URL_PATH} doca_base_src RUN set -eux && \ if [ "${D_OFED_URL_PATH}" = "README.md" ]; then \ src="${D_OFED_URL_DEFAULT}" && \ legacy="${D_OFED_BASE_URL}/MLNX_OFED_SRC-${D_OFED_SRC_TYPE}${D_OFED_VERSION}.tgz" && \ doca_major=$(printf '%s' "${D_DOCA_VERSION}" | cut -d. -f1) && \ doca_minor=$(printf '%s' "${D_DOCA_VERSION}" | cut -d. -f2 | cut -d- -f1) && \ doca_base=$(printf '%s' "${D_DOCA_VERSION}" | cut -d- -f1) && \ if [ "$src" = "$legacy" ] && { [ "$doca_major" -gt 3 ] || { [ "$doca_major" -eq 3 ] && [ "$doca_minor" -ge 6 ]; }; }; then \ case "${D_DOCA_VERSION}" in \ *-*) doca_full="${D_DOCA_VERSION}" ;; \ *) \ ofed_build=$(printf '%s' "${D_OFED_VERSION}" | cut -d- -f2-) && \ case "$ofed_build" in \ ''|*[!0-9]*) \ echo "error: DOCA ${D_DOCA_VERSION} archives are named DOCA_BASE_SRC-${D_OFED_SRC_TYPE}${doca_base}-.tar.gz; set D_OFED_VERSION to - (for example 26.10-069000) or pass D_OFED_URL_PATH" >&2 && \ exit 1 ;; \ *) doca_full="${doca_base}-${ofed_build}" ;; \ esac ;; \ esac && \ base=$(printf '%s' "${D_OFED_BASE_URL}" | sed "s#/doca/${D_DOCA_VERSION}/#/doca/${doca_base}/#") && \ src="${base}/DOCA_BASE_SRC-${D_OFED_SRC_TYPE}${doca_full}.tar.gz" ; \ fi && \ echo "Using OFED source: ${src}" && \ printf '%s\n' "$src" > ofed_src_resolved && \ curl -fsSL -o doca_base_src "$src" ; \ else \ echo "Using OFED source: ${D_OFED_URL_PATH}" && \ printf '%s\n' "${D_OFED_URL_PATH}" > ofed_src_resolved ; \ fi # Up to DOCA 3.5 the archive extracts to MLNX_OFED_SRC-/ complete with # install.pl, landing directly at NVIDIA_NIC_DRIVER_PATH with no further work needed. # DOCA 3.6 deprecated install.pl and ships a flat DOCA_BASE_SRC-* directory of upstream # tarballs instead, so that layout is reshaped into the tree the build expects and the # vendored scripts are staged into it. See vendor/mlnx_ofed/README.md. COPY vendor/mlnx_ofed /opt/mlnx_ofed RUN set -eux && \ resolved=$(cat ofed_src_resolved 2>/dev/null || printf '%s' "${D_OFED_URL_PATH}") && \ rm -f ofed_src_resolved && \ if [ -d doca_base_src ]; then \ mv doca_base_src/* . && rmdir doca_base_src; \ else \ tar -xzf doca_base_src && rm -f doca_base_src; \ fi && \ src_dir=$(find . -maxdepth 1 -type d -name 'DOCA_BASE_SRC-*' | head -1) && \ if [ -n "${src_dir}" ]; then \ /opt/mlnx_ofed/stage_src_tree.sh "${src_dir}" "${NVIDIA_NIC_DRIVER_PATH}" deb; \ elif [ -f "${NVIDIA_NIC_DRIVER_PATH}/install.pl" ]; then \ echo "Using legacy source tree with bundled install.pl: ${NVIDIA_NIC_DRIVER_PATH}"; \ else \ echo "error: ${resolved} yielded neither a DOCA_BASE_SRC-* directory nor ${NVIDIA_NIC_DRIVER_PATH}/install.pl" >&2; \ exit 1; \ fi WORKDIR /root CMD ["sources"] LABEL doca-version=${D_DOCA_VERSION} LABEL ofed-version=${D_OFED_VERSION} ##################### # Stage: Build driver FROM driver-src AS driver-builder # Inherited global args ARG D_OS ARG D_KERNEL_VER ARG OFED_SRC_LOCAL_DIR # Additional local arg (for precompiled CI) ARG D_BUILD_EXTRA_ARGS ARG D_ENABLE_DKMS # Driver build manadatory packages RUN set -x && \ apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get -yq -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install linux-image-${D_KERNEL_VER} linux-headers-${D_KERNEL_VER} # Build driver. When D_ENABLE_DKMS=true, install.pl produces DKMS-enabled .deb packages # (source + dkms.conf under /usr/src/-/). When false, --without-dkms is used. # Only the exact string "true" enables DKMS; reject other values so they cannot produce # non-DKMS packages while the runtime bool parser treats them as enabled. RUN set -x && \ case "$D_ENABLE_DKMS" in \ true|false) ;; \ *) echo "error: D_ENABLE_DKMS must be exactly 'true' or 'false', got '$D_ENABLE_DKMS'" >&2; exit 1 ;; \ esac && \ if [ "$D_ENABLE_DKMS" = "true" ]; then \ ${OFED_SRC_LOCAL_DIR}/install.pl --without-depcheck --distro ${D_OS} --kernel ${D_KERNEL_VER} --kernel-only --build-only --copy-ifnames-udev --with-mlnx-tools --without-knem-modules --without-srp-modules --without-kernel-mft-modules --without-iser-modules --without-isert-modules ${D_BUILD_EXTRA_ARGS}; \ else \ ${OFED_SRC_LOCAL_DIR}/install.pl --without-depcheck --distro ${D_OS} --without-dkms --kernel ${D_KERNEL_VER} --kernel-only --build-only --copy-ifnames-udev --with-mlnx-tools --without-knem-modules --without-srp-modules --without-kernel-mft-modules --without-iser-modules --without-isert-modules ${D_BUILD_EXTRA_ARGS}; \ fi ################################### # Stage: Install precompiled driver FROM base AS precompiled # Inherited global args ARG D_OS ARG D_ARCH ARG D_KERNEL_VER ARG OFED_SRC_LOCAL_DIR ARG D_ENABLE_DKMS ENV NVIDIA_NIC_DRIVER_PATH="" # Canonical true/false matching install.pl ([ "$D_ENABLE_DKMS" = "true" ]). # Reject TRUE/1/yes so Go cannot parse them as enabled against non-DKMS packages. RUN case "$D_ENABLE_DKMS" in \ true|false) ;; \ *) echo "error: D_ENABLE_DKMS must be exactly 'true' or 'false', got '$D_ENABLE_DKMS'" >&2; exit 1 ;; \ esac ENV NVIDIA_NIC_DRIVER_DKMS_ENABLED=${D_ENABLE_DKMS} RUN set -x && \ DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" lsb-release && \ test -n "${D_KERNEL_VER}" && DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" linux-modules-extra-${D_KERNEL_VER} || true && \ # Install DKMS and build tools if enabled (needed at runtime for dkms add/build/install) if [ "$D_ENABLE_DKMS" = "true" ]; then \ DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dkms build-essential bison flex libelf-dev pkg-config zstd; \ fi # Cleanup RUN set -x && \ apt-get clean autoclean && \ rm -rf /var/lib/apt/lists/* # Install driver packages. When D_ENABLE_DKMS was true, .debs install source and dkms.conf # under /usr/src/. Runtime DKMS setup (add/build/install) runs in entrypoint.sh and driver.go. COPY --from=driver-builder ${OFED_SRC_LOCAL_DIR}/DEBS/${D_OS}/*/*.deb /root/ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" /root/*.deb # Prevent modprobe from giving a WARNING about missing files RUN touch /lib/modules/${D_KERNEL_VER}/modules.order /lib/modules/${D_KERNEL_VER}/modules.builtin && \ # Introduce installed kernel modules depmod ${D_KERNEL_VER} CMD ["precompiled"]