diff --git a/.github/workflows/ci2.yml b/.github/workflows/ci2.yml index 9776a4c6306..d133dbf56f3 100644 --- a/.github/workflows/ci2.yml +++ b/.github/workflows/ci2.yml @@ -134,6 +134,7 @@ jobs: --build-arg BASE_DISTRO --build-arg DAOS_JAVA_BUILD=yes --build-arg COMPILER + --target build-local - name: Build debug in docker. run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -141,6 +142,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=debug --build-arg COMPILER + --target build-local - name: Build devel in docker. run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -148,6 +150,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=dev --build-arg COMPILER + --target build-local # Should work, but enable on master only for now. # - name: Run NLT diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index b3265275771..323e0f5de73 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -88,12 +88,14 @@ jobs: --build-arg DEPS_JOBS --build-arg BASE_DISTRO --build-arg PYTHON_VERSION + --target build-local - name: Build dependencies in Docker run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DAOS_BUILD=no --build-arg DEPS_JOBS --build-arg BASE_DISTRO --build-arg PYTHON_VERSION + --target build-local - name: Prune images not required for build. run: docker images --all --filter label=DAOS=true --quiet | xargs docker rmi --no-prune @@ -227,6 +229,7 @@ jobs: --build-arg PYTHON_VERSION --build-arg DAOS_JAVA_BUILD=yes --build-arg COMPILER + --target build-local - name: Build debug in docker. run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -235,6 +238,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=debug --build-arg COMPILER + --target build-local - name: Build devel in docker. run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -243,6 +247,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=dev --build-arg COMPILER + --target build-local # Fails with Ubuntu still for the spdk issue. # - name: Run NLT # run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 @@ -289,6 +294,7 @@ jobs: --build-arg BASE_DISTRO --build-arg PYTHON_VERSION --build-arg DAOS_BUILD=no + --target build-local - name: Build in docker with clang run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -305,6 +311,7 @@ jobs: --build-arg PYTHON_VERSION --build-arg DAOS_JAVA_BUILD=yes --build-arg COMPILER=clang + --target build-local - name: Build debug in docker with clang. run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -313,6 +320,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=debug --build-arg COMPILER=clang + --target build-local - name: Build devel in docker with clang run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -321,6 +329,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=dev --build-arg COMPILER=clang + --target build-local - name: Build in docker with gcc run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -328,6 +337,7 @@ jobs: --build-arg PYTHON_VERSION --build-arg DAOS_JAVA_BUILD=no --build-arg COMPILER=gcc + --target build-local - name: Build debug in docker with gcc run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -336,6 +346,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=debug --build-arg COMPILER=gcc + --target build-local - name: Build devel in docker with gcc run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -344,6 +355,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=dev --build-arg COMPILER=gcc + --target build-local - name: Run NLT run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 --user root:root build-image ./daos/utils/node_local_test.py --no-root @@ -390,6 +402,7 @@ jobs: --build-arg BASE_DISTRO --build-arg PYTHON_VERSION --build-arg DAOS_BUILD=no + --target build-local - name: Build in docker with clang run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -406,6 +419,7 @@ jobs: --build-arg PYTHON_VERSION --build-arg DAOS_JAVA_BUILD=yes --build-arg COMPILER=clang + --target build-local - name: Build debug in docker with clang. run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -414,6 +428,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=debug --build-arg COMPILER=clang + --target build-local - name: Build devel in docker with clang run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -422,6 +437,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=dev --build-arg COMPILER=clang + --target build-local - name: Build in docker with gcc run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -429,6 +445,7 @@ jobs: --build-arg PYTHON_VERSION --build-arg DAOS_JAVA_BUILD=no --build-arg COMPILER=gcc + --target build-local - name: Build debug in docker with gcc run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -437,6 +454,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=debug --build-arg COMPILER=gcc + --target build-local - name: Build devel in docker with gcc run: docker build . --file utils/docker/Dockerfile.${{ matrix.base }} --build-arg DEPS_JOBS @@ -445,6 +463,7 @@ jobs: --build-arg DAOS_JAVA_BUILD=no --build-arg DAOS_BUILD_TYPE=dev --build-arg COMPILER=gcc + --target build-local - name: Run NLT run: docker run --mount type=tmpfs,destination=/mnt/daos_0,tmpfs-mode=1777 --user root:root build-image ./daos/utils/node_local_test.py --no-root diff --git a/Jenkinsfile b/Jenkinsfile index 357f489d6a6..ca433f58869 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -903,8 +903,7 @@ pipeline { deps_build: false, parallel_build: true) + " -t ${sanitized_JOB_NAME()}-el9 " + - ' --build-arg DAOS_PACKAGES_BUILD=no ' + - ' --build-arg DAOS_KEEP_SRC=yes ' + + ' --target build-ci' + ' --build-arg REPOS="' + prRepos() + '"' + ' --build-arg POINT_RELEASE=.7' + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" @@ -970,10 +969,8 @@ pipeline { additionalBuildArgs dockerBuildArgs(repo_type: 'stable', parallel_build: true, deps_build: false) + - ' --build-arg DAOS_PACKAGES_BUILD=no ' + - ' --build-arg DAOS_KEEP_SRC=yes ' + - " -t ${sanitized_JOB_NAME()}-leap15-gcc" + " -t ${sanitized_JOB_NAME()}-leap15" + + ' --target build-ci' + ' --build-arg POINT_RELEASE=.6' + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" } diff --git a/ci/rpm/build_deps.sh b/ci/rpm/build_deps.sh index f989a96332a..06638601d0b 100755 --- a/ci/rpm/build_deps.sh +++ b/ci/rpm/build_deps.sh @@ -1,3 +1,2 @@ #!/bin/bash -cd /home/daos/pre || exit 1 scons install --build-deps=only USE_INSTALLED=all PREFIX=/opt/daos TARGET_TYPE=release -j 32 diff --git a/utils/docker/Dockerfile.el.8 b/utils/docker/Dockerfile.el.8 index 8eef34f4cf7..fa27e6b13e4 100644 --- a/utils/docker/Dockerfile.el.8 +++ b/utils/docker/Dockerfile.el.8 @@ -41,7 +41,7 @@ RUN chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ /tmp/repo-helper.sh && \ rm -f /tmp/repo-helper.sh -FROM basic +FROM basic AS build-local # Install OS updates and package. Include basic tools and daos dependencies # The proxy variables are currently needed for the combination of running # with a local repository, yet needing a proxy to reach outside repositories. diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index ac1a99798fc..f3a77a614a8 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -6,6 +6,7 @@ # 'recipe' for Docker to build an image of EL 9 based # environment for building the DAOS project. # +# Use --target build-ci to create image with basic build environment for gcc # Pull base image ARG POINT_RELEASE= @@ -15,7 +16,7 @@ LABEL maintainer="daos@daos.groups.io" # Needed for later use of BASE_DISTRO ARG BASE_DISTRO -# Intermittent cache-bust. Used to reduce load on the actual CB1 later. +# Intermittent cache-bust. Refresh base image at least once a week. ARG CB0 ARG REPO_FILE_URL @@ -32,60 +33,82 @@ ENV NO_PROXY=${DAOS_NO_PROXY} RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \ echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment -# script to install OS updates basic tools and daos dependencies -COPY ./utils/scripts/install-el9.sh /tmp/install.sh -# script to setup local repo if available +# Script to setup local repo if available COPY ./utils/scripts/helpers/repo-helper-el9.sh /tmp/repo-helper.sh -RUN chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ - /tmp/repo-helper.sh && \ +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ rm -f /tmp/repo-helper.sh -FROM basic -# Install OS updates and package. Include basic tools and daos dependencies -RUN dnf upgrade && \ - /tmp/install.sh && \ - dnf clean all && \ - rm -f /tmp/install.sh +# Install OS updates. +RUN dnf upgrade && \ + dnf clean all + +# Setup docker image for gcc build only +FROM basic AS build-ci +# Script to install basic tools and DAOS dependencies +COPY ./utils/scripts/install-el9.sh /tmp/install.sh + +# Install basic tools and DAOS dependencies +RUN dnf upgrade && \ + chmod +x /tmp/install.sh && \ + INSTALL_BUILD_CI_ONLY="true" /tmp/install.sh && \ + dnf clean all -# Add DAOS users +# Add DAOS user ARG UID=1000 COPY ./utils/scripts/helpers/daos-server-user-setup.sh \ /tmp/daos-server-user-setup.sh RUN set -e; \ chmod +x /tmp/daos-server-user-setup.sh && \ - /tmp/daos-server-user-setup.sh -RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent -RUN echo "daos_agent:daos_agent" | chpasswd + /tmp/daos-server-user-setup.sh && \ + rm -f /tmp/daos-server-user-setup.sh -# Create directory for DAOS backend storage -RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent /home/daos/pre /home/daos/daos && \ - chown -R daos_server.daos_server /opt/daos /mnt/daos /var/run/daos_server /home/daos && \ - chown daos_agent.daos_agent /var/run/daos_agent - -USER daos_server:daos_server +# Create directory for DAOS build backend storage +RUN mkdir -p /home/daos /opt/daos && \ + chown -R daos_server.daos_server /home/daos /opt/daos # Setup a python venv so that python packages can be installed locally. +USER daos_server:daos_server ARG PYTHON_VERSION=3.11 RUN python${PYTHON_VERSION} -m venv /home/daos/venv ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. -COPY requirements-build.txt requirements-utest.txt ./ -RUN . /home/daos/venv/bin/activate && \ - pip --no-cache-dir install --upgrade pip && \ - pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt +WORKDIR /home/daos +COPY requirements-build.txt ./ +RUN . /home/daos/venv/bin/activate && \ + pip --no-cache-dir install --upgrade pip && \ + pip --no-cache-dir install -r requirements-build.txt && \ + rm -r requirements-build.txt + +# Setup docker image for build using local copy of sources. +# Include also clang and java compilers installation +FROM build-ci AS build-local +# Install remaining tools and dependencies needed for internal build. +# This is done in a separate stage to allow for a smaller image for CI building only. +USER root:root +RUN /tmp/install.sh && \ + dnf clean all && \ + rm -f /tmp/install.sh + +# Create directory for DAOS dependencies build storage +USER daos_server:daos_server +RUN mkdir -p /home/daos/pre/site_scons/prereq_tools \ + /home/daos/pre/site_scons/components WORKDIR /home/daos/pre -RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components COPY --chown=daos_server:daos_server SConstruct . COPY --chown=daos_server:daos_server deps deps -COPY --chown=daos_server:daos_server site_scons/prereq_tools site_scons/prereq_tools +COPY --chown=daos_server:daos_server site_scons/prereq_tools \ + site_scons/prereq_tools COPY --chown=daos_server:daos_server site_scons/components site_scons/components COPY --chown=daos_server:daos_server utils/build.config utils/ -COPY --chown=daos_server:daos_server utils/scripts/copy_files.sh utils/scripts/copy_files.sh -COPY --chown=daos_server:daos_server utils/scripts/create_spdk_pkgconfig.sh utils/scripts/create_spdk_pkgconfig.sh +COPY --chown=daos_server:daos_server utils/scripts/copy_files.sh \ + utils/scripts/copy_files.sh +COPY --chown=daos_server:daos_server utils/scripts/create_spdk_pkgconfig.sh \ + utils/scripts/create_spdk_pkgconfig.sh # Control what to build. By default Dockerfiles build everything to allow for # ease-of-use for users, however in CI everything is turned off and then @@ -99,12 +122,14 @@ ARG DAOS_PACKAGES_BUILD=yes # src hasn't changed then this won't do anything, but if it has then we want to # ensure that latest dependencies are used. USER root:root -RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ - dnf upgrade --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \ - dnf clean all; \ +RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ + dnf upgrade \ + --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \ + dnf clean all; \ } -USER daos_server:daos_server +# Build dependencies +USER daos_server:daos_server ARG DEPS_JOBS=1 RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ @@ -116,29 +141,34 @@ RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ COPY --chown=daos_server:daos_server utils/rpms utils/rpms COPY --chown=daos_server:daos_server utils/sl utils/sl -# Build third party RPMs -RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ - export DISTRO="el9" && \ - utils/rpms/build_packages.sh deps && \ - mkdir -p /home/daos/rpms && \ - mv *.rpm /home/daos/rpms; \ +# Build dependencies RPMs +RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || \ + { \ + export DISTRO="el9" && \ + utils/rpms/build_packages.sh deps && \ + mkdir -p /home/daos/rpms/deps && \ + mv *.rpm /home/daos/rpms/deps; \ } -USER root:root # force an upgrade to get any newly built RPMs, but only if CB1 is set. ARG CB1 -RUN [ -z "$CB1" ] || { \ - dnf upgrade --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \ - dnf clean all; \ +USER root:root +RUN [ -z "$CB1" ] || { \ + dnf upgrade \ + --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \ + dnf clean all; \ } -USER daos_server:daos_server # Set a label. This is useful for searching for DAOS images, but is also used # in github-actions to prune elements of the dockerfile below this point. LABEL DAOS=true +# Create directory for DAOS dependencies build storage +USER daos_server:daos_server +RUN mkdir -p /home/daos/daos WORKDIR /home/daos/daos/ -COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-ftest.txt .clang-format ./ +COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct \ + requirements-ftest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons COPY --chown=daos_server:daos_server src src COPY --chown=daos_server:daos_server utils/build.config utils/ @@ -153,35 +183,30 @@ ARG DAOS_BUILD_TYPE=$DAOS_TARGET_TYPE ARG DAOS_BUILD=$DAOS_DEPS_BUILD # Build DAOS -RUN [ "$DAOS_BUILD" != "yes" ] || { \ - scons --jobs $JOBS install PREFIX=/opt/daos COMPILER=$COMPILER \ - FIRMWARE_MGMT=1 BUILD_TYPE=$DAOS_BUILD_TYPE \ - TARGET_TYPE=$DAOS_TARGET_TYPE && \ - ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build) && \ - go clean -cache && \ - cp -r utils/config/examples /opt/daos; \ +RUN [ "$DAOS_BUILD" != "yes" ] || { \ + scons --jobs $JOBS install PREFIX=/opt/daos COMPILER=$COMPILER \ + BUILD_TYPE=$DAOS_BUILD_TYPE TARGET_TYPE=$DAOS_TARGET_TYPE \ + FIRMWARE_MGMT=1 && \ + ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build) && \ + go clean -cache && \ + cp -r utils/config/examples /opt/daos; \ } COPY --chown=daos_server:daos_server utils utils # Build DAOS RPMs RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ - export DISTRO="el9" && \ - utils/rpms/build_packages.sh daos && \ - mkdir -p /home/daos/rpms && \ - cp *.rpm /home/daos/rpms; \ + export DISTRO="el9" && \ + utils/rpms/build_packages.sh daos && \ + mkdir -p /home/daos/rpms/daos && \ + mv *.rpm /home/daos/rpms/daos; \ } -# Set environment variables -ENV PATH=/opt/daos/bin:$PATH -ENV FI_SOCKETS_MAX_CONN_RETRY=1 - -# Build java and hadoop bindings -WORKDIR /home/daos/daos/src/client/java - -ARG DAOS_JAVA_BUILD=$DAOS_BUILD - # Disable Java build for now since it fails +# Build java and hadoop bindings +#WORKDIR /home/daos/daos/src/client/java +# +#ARG DAOS_JAVA_BUILD=$DAOS_BUILD #RUN [ "$DAOS_JAVA_BUILD" != "yes" ] || { \ # mkdir /home/daos/.m2 && \ # cp /home/daos/daos/utils/scripts/helpers/maven-settings.xml.in /home/daos/.m2/settings.xml && \ @@ -189,8 +214,32 @@ ARG DAOS_JAVA_BUILD=$DAOS_BUILD # -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ # -DskipITs -Dgpg.skip -Ddaos.install.path=/opt/daos; \ # } -WORKDIR /home/daos -ARG DAOS_KEEP_SRC=no # Remove local copy +ARG DAOS_KEEP_SRC=no RUN [ "$DAOS_KEEP_SRC" != "no" ] || rm -rf /home/daos/daos /home/daos/pre + +# Setup docker image for Unit Tests +FROM build-local AS full + +# Add DAOS agent users +USER root:root +RUN useradd --no-log-init --user-group --create-home --shell /bin/bash \ + daos_agent && \ + echo "daos_agent:daos_agent" | chpasswd + +# Create directory for DAOS runtime +RUN mkdir -p /mnt/daos /var/run/daos_server /var/run/daos_agent && \ + chown -R daos_server.daos_server /mnt/daos /var/run/daos_server && \ + chown -R daos_agent.daos_agent /var/run/daos_agent + +# Update python virtual environment for unit tests +WORKDIR /home/daos +USER daos_server:daos_server +COPY requirements-utest.txt ./ +RUN pip --no-cache-dir install -r requirements-utest.txt && \ + rm -f requirements-utest.txt + +# Set environment variables +ENV PATH=/opt/daos/bin:$PATH +ENV FI_SOCKETS_MAX_CONN_RETRY=1 diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15 index a9c2129d9f2..350857f2b65 100644 --- a/utils/docker/Dockerfile.leap.15 +++ b/utils/docker/Dockerfile.leap.15 @@ -6,6 +6,7 @@ # 'recipe' for Docker to build an image of Leap based # environment for building the DAOS project. # +# Use --target build-ci to create image with basic build environment for gcc # Pull base image ARG POINT_RELEASE= @@ -15,7 +16,7 @@ LABEL maintainer="daos@daos.groups.io" # Needed for later use of BASE_DISTRO ARG BASE_DISTRO -# Intermittent cache-bust. Used to reduce load on the actual CB1 later. +# Intermittent cache-bust. Refresh base image at least once a week. ARG CB0 ARG REPO_FILE_URL @@ -32,26 +33,27 @@ ENV NO_PROXY=${DAOS_NO_PROXY} RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \ echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment -# script to install OS updates basic tools and daos dependencies -COPY ./utils/scripts/install-leap15.sh /tmp/install.sh -# script to setup local repo if available +# Script to setup local repo if available COPY ./utils/scripts/helpers/repo-helper-leap15.sh /tmp/repo-helper.sh -RUN chmod +x /tmp/repo-helper.sh /tmp/install.sh && \ - /tmp/repo-helper.sh && \ +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ rm -f /tmp/repo-helper.sh -FROM basic -# Install OS updates and package. Include basic tools and daos dependencies -# Install OS updates and package. Include basic tools and daos dependencies -# The unset commands are currently needed for the combination of running -# with a local repository, yet needing a proxy to reach outside repositories. -# This needs to be moved to a shell script like above in the future to -# properly only remove the proxy variables only when they need to be removed -RUN dnf upgrade && \ - /tmp/install.sh && \ - dnf clean all && \ - rm -f /tmp/install.sh +# Install OS updates. +RUN dnf upgrade && \ + dnf clean all + +# Setup docker image for gcc build only +FROM basic AS build-ci +# Script to install basic tools and DAOS dependencies +COPY ./utils/scripts/install-leap15.sh /tmp/install.sh + +# Install basic tools and DAOS dependencies +RUN dnf upgrade && \ + chmod +x /tmp/install.sh && \ + INSTALL_BUILD_CI_ONLY="true" /tmp/install.sh && \ + dnf clean all # According to https://pkgs.org/search/?q=lua-lmod, Leap 15.6 only has lua-lmod-8.7.34. # This version has a problem with loading modules, as described in https://github.com/TACC/Lmod/issues/687. @@ -62,49 +64,65 @@ RUN dnf upgrade && \ # opensuse-oss for lua53, lua53-luaterm,, sqlite3-tcl, tcl # opensuse-devel-languages-lua for lua53-luafilesystem, lua53-luaposix ARG JENKINS_URL -RUN if [ -n "$JENKINS_URL" ]; then \ - dnf -y remove lua-lmod; \ +RUN if [ -n "$JENKINS_URL" ]; then \ + dnf -y remove lua-lmod; \ dnf -y --nogpgcheck install lua-lmod --repo '*lua*' --repo '*network-cluster*' --repo '*oss-proxy*'; \ fi - + # Add DAOS users ARG UID=1000 COPY ./utils/scripts/helpers/daos-server-user-setup.sh \ /tmp/daos-server-user-setup.sh RUN set -e; \ chmod +x /tmp/daos-server-user-setup.sh && \ - /tmp/daos-server-user-setup.sh -RUN useradd --no-log-init --user-group --create-home --shell /bin/bash daos_agent -RUN echo "daos_agent:daos_agent" | chpasswd + /tmp/daos-server-user-setup.sh && \ + rm -f /tmp/daos-server-user-setup.sh -# Create directory for DAOS backend storage -RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent /home/daos/pre /home/daos/daos && \ - chown -R daos_server.daos_server /opt/daos /mnt/daos /var/run/daos_server /home/daos && \ - chown daos_agent.daos_agent /var/run/daos_agent - -USER daos_server:daos_server +# Create directory for DAOS build backend storage +RUN mkdir -p /home/daos /opt/daos && \ + chown -R daos_server.daos_server /home/daos /opt/daos # Setup a python venv so that python packages can be installed locally. +USER daos_server:daos_server ARG PYTHON_VERSION=3.11 RUN python${PYTHON_VERSION} -m venv /home/daos/venv ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. -COPY requirements-build.txt requirements-utest.txt ./ -RUN . /home/daos/venv/bin/activate && \ - pip --no-cache-dir install --upgrade pip && \ - pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt +WORKDIR /home/daos +COPY requirements-build.txt ./ +RUN . /home/daos/venv/bin/activate && \ + pip --no-cache-dir install --upgrade pip && \ + pip --no-cache-dir install -r requirements-build.txt && \ + rm -r requirements-build.txt + +# Setup docker image for build using local copy of sources. +# Include also clang and java compilers installation +FROM build-ci AS build-local +# Install remaining tools and dependencies needed for internal build. +# This is done in a separate stage to allow for a smaller image for CI building only. +USER root:root +RUN /tmp/install.sh && \ + dnf clean all && \ + rm -f /tmp/install.sh + +# Create directory for DAOS dependencies build storage +USER daos_server:daos_server +RUN mkdir -p /home/daos/pre/site_scons/prereq_tools \ + /home/daos/pre/site_scons/components WORKDIR /home/daos/pre -RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components COPY --chown=daos_server:daos_server SConstruct . COPY --chown=daos_server:daos_server deps deps -COPY --chown=daos_server:daos_server site_scons/prereq_tools site_scons/prereq_tools +COPY --chown=daos_server:daos_server site_scons/prereq_tools \ + site_scons/prereq_tools COPY --chown=daos_server:daos_server site_scons/components site_scons/components COPY --chown=daos_server:daos_server utils/build.config utils/ -COPY --chown=daos_server:daos_server utils/scripts/copy_files.sh utils/scripts/copy_files.sh -COPY --chown=daos_server:daos_server utils/scripts/create_spdk_pkgconfig.sh utils/scripts/create_spdk_pkgconfig.sh +COPY --chown=daos_server:daos_server utils/scripts/copy_files.sh \ + utils/scripts/copy_files.sh +COPY --chown=daos_server:daos_server utils/scripts/create_spdk_pkgconfig.sh \ + utils/scripts/create_spdk_pkgconfig.sh # Control what to build. By default Dockerfiles build everything to allow for # ease-of-use for users, however in CI everything is turned off and then @@ -117,14 +135,15 @@ ARG DAOS_PACKAGES_BUILD=yes # Now do an update to ensure software is up to date for the deps build. If the # src hasn't changed then this won't do anything, but if it has then we want to # ensure that latest dependencies are used. -# The dnf upgrade can add or re-enable distro repositories. USER root:root -RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ - dnf upgrade --exclude=fuse,fuse-libs,fuse-devel,libraft0,raft-devel,mercury,mercury-devel && \ - dnf clean all; \ +RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ + dnf upgrade \ + --exclude=fuse,fuse-libs,fuse-devel,libraft0,raft-devel,mercury,mercury-devel && \ + dnf clean all; \ } -USER daos_server:daos_server +# Build dependencies +USER daos_server:daos_server ARG DEPS_JOBS=1 RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ @@ -136,40 +155,34 @@ RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ COPY --chown=daos_server:daos_server utils/rpms utils/rpms COPY --chown=daos_server:daos_server utils/sl utils/sl -# Build third party RPMs -RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ - export DISTRO="suse.lp15${POINT_RELEASE#\.}" && \ - utils/rpms/build_packages.sh deps && \ - mkdir -p /home/daos/rpms && \ - mv *.rpm /home/daos/rpms; \ +# Build dependencies RPMs +RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || \ + { \ + export DISTRO="suse.lp15${POINT_RELEASE#\.}" && \ + utils/rpms/build_packages.sh deps && \ + mkdir -p /home/daos/rpms/deps && \ + mv *.rpm /home/daos/rpms/deps; \ } -USER root:root -# select compiler to use -# Load the COMPILER arg early, and optionally install the Intel compiler -# Do this before the CB1 to make best use of cache as this is a big -# download, and make it optional on the value of COMPILER for the same -# reason. Tell zypper to only consider the oneAPI repo here using the -# discouraged --repo flag, however this prevents an update of the entire -# upstream package list which would be immediately discarded, then reloaded -# below in the general update section. Only install the compilers at -# ~900Mb rather than the entire stack at ~3Gb download. -ARG COMPILER=gcc - -# force an upgrade to get any newly built RPMs, but only if CB1 is set. + # force an upgrade to get any newly built RPMs, but only if CB1 is set. ARG CB1 -RUN [ -z "$CB1" ] || { \ - dnf upgrade --exclude=fuse,fuse-libs,fuse-devel,libraft0,raft-devel,mercury,mercury-devel && \ - dnf clean all; \ +USER root:root +RUN [ -z "$CB1" ] || { \ + dnf upgrade \ + --exclude=fuse,fuse-libs,fuse-devel,libraft0,raft-devel,mercury,mercury-devel && \ + dnf clean all; \ } -USER daos_server:daos_server # Set a label. This is useful for searching for DAOS images, but is also used # in github-actions to prune elements of the dockerfile below this point. LABEL DAOS=true +# Create directory for DAOS dependencies build storage +USER daos_server:daos_server +RUN mkdir -p /home/daos/daos WORKDIR /home/daos/daos/ -COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-ftest.txt .clang-format ./ +COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct \ + requirements-ftest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons COPY --chown=daos_server:daos_server src src COPY --chown=daos_server:daos_server utils/build.config utils/ @@ -177,46 +190,69 @@ COPY --chown=daos_server:daos_server utils/config utils/config COPY --chown=daos_server:daos_server utils/certs utils/certs COPY --chown=daos_server:daos_server utils/completion utils/completion +# select compiler to use +ARG COMPILER=gcc ARG JOBS=$DEPS_JOBS ARG DAOS_BUILD_TYPE=$DAOS_TARGET_TYPE ARG DAOS_BUILD=$DAOS_DEPS_BUILD # Build DAOS -RUN [ "$DAOS_BUILD" != "yes" ] || { \ - scons --jobs $JOBS install PREFIX=/opt/daos COMPILER=$COMPILER \ - BUILD_TYPE=$DAOS_BUILD_TYPE TARGET_TYPE=$DAOS_TARGET_TYPE \ - FIRMWARE_MGMT=1 && \ - ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build) && \ - go clean -cache && \ - cp -r utils/config/examples /opt/daos; \ +RUN [ "$DAOS_BUILD" != "yes" ] || { \ + scons --jobs $JOBS install PREFIX=/opt/daos COMPILER=$COMPILER \ + BUILD_TYPE=$DAOS_BUILD_TYPE TARGET_TYPE=$DAOS_TARGET_TYPE \ + FIRMWARE_MGMT=1 && \ + ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build) && \ + go clean -cache && \ + cp -r utils/config/examples /opt/daos; \ } COPY --chown=daos_server:daos_server utils utils # Build DAOS RPMs RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ - export DISTRO="suse.lp15${POINT_RELEASE#\.}" && \ - utils/rpms/build_packages.sh daos && \ - mkdir -p /home/daos/rpms && \ - cp *.rpm /home/daos/rpms; \ + export DISTRO="suse.lp15${POINT_RELEASE#\.}" && \ + utils/rpms/build_packages.sh daos && \ + mkdir -p /home/daos/rpms/daos && \ + mv *.rpm /home/daos/rpms/daos; \ } -# Set environment variables -ENV PATH=/opt/daos/bin:$PATH -ENV FI_SOCKETS_MAX_CONN_RETRY=1 - # Build java and hadoop bindings WORKDIR /home/daos/daos/src/client/java ARG DAOS_JAVA_BUILD=$DAOS_BUILD - -RUN [ "$DAOS_JAVA_BUILD" != "yes" ] || { \ - mkdir /home/daos/.m2 && \ - cp /home/daos/daos/utils/scripts/helpers/maven-settings.xml.in /home/daos/.m2/settings.xml && \ - mvn clean install -ntp -T 1C -DskipITs -Dgpg.skip -Ddaos.install.path=/opt/daos; \ +RUN [ "$DAOS_JAVA_BUILD" != "yes" ] || { \ + mkdir /home/daos/.m2 && \ + cp /home/daos/daos/utils/scripts/helpers/maven-settings.xml.in \ + /home/daos/.m2/settings.xml && \ + mvn clean install -ntp -T 1C -DskipITs -Dgpg.skip \ + -Ddaos.install.path=/opt/daos; \ } -WORKDIR /home/daos -ARG DAOS_KEEP_SRC=no # Remove local copy +ARG DAOS_KEEP_SRC=no RUN [ "$DAOS_KEEP_SRC" != "no" ] || rm -rf /home/daos/daos /home/daos/pre + +# Setup docker image for Unit Tests +FROM build-local AS full + +# Add DAOS agent users +USER root:root +RUN useradd --no-log-init --user-group --create-home --shell /bin/bash \ + daos_agent && \ + echo "daos_agent:daos_agent" | chpasswd + +# Create directory for DAOS runtime +RUN mkdir -p /mnt/daos /var/run/daos_server /var/run/daos_agent && \ + chown -R daos_server.daos_server /mnt/daos /var/run/daos_server && \ + chown -R daos_agent.daos_agent /var/run/daos_agent + +# Update python virtual environment for unit tests +WORKDIR /home/daos +USER daos_server:daos_server +COPY requirements-utest.txt ./ +RUN pip --no-cache-dir install -r requirements-utest.txt && \ + rm -f requirements-utest.txt + +# Set environment variables +ENV PATH=/opt/daos/bin:$PATH +ENV FI_SOCKETS_MAX_CONN_RETRY=1 diff --git a/utils/docker/Dockerfile.ubuntu b/utils/docker/Dockerfile.ubuntu index b65e3eddad8..196ce4dda74 100644 --- a/utils/docker/Dockerfile.ubuntu +++ b/utils/docker/Dockerfile.ubuntu @@ -10,7 +10,7 @@ # Pull base image ARG BASE_DISTRO=ubuntu:24.04 -FROM $BASE_DISTRO +FROM $BASE_DISTRO AS build-local LABEL maintainer="daos@daos.groups.io" # Needed for later use of BASE_DISTRO ARG BASE_DISTRO diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index 8f32803071c..54372ac91dd 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # (C) Copyright 2025 Google LLC +# Copyright 2026 Hewlett Packard Enterprise Development LP # Install OS updates and packages as required for building DAOS on EL 9 and # derivatives. Include basic tools and daos dependencies that come from the core repos. @@ -20,8 +21,6 @@ dnf --nodocs install ${dnf_install_args} \ boost-python3-devel \ bzip2 \ capstone-devel \ - clang \ - clang-tools-extra \ cmake \ createrepo \ CUnit-devel \ @@ -31,19 +30,14 @@ dnf --nodocs install ${dnf_install_args} \ fdupes \ file \ flex \ - fuse3 \ gcc \ gcc-c++ \ git \ glibc-langpack-en \ golang \ - gperftools-devel \ - graphviz \ help2man \ hdf5-devel \ hwloc-devel \ - ipmctl \ - java-1.8.0-openjdk \ json-c-devel \ libaio-devel \ libasan \ @@ -64,28 +58,40 @@ dnf --nodocs install ${dnf_install_args} \ Lmod \ make \ nasm \ - ndctl \ ndctl-devel \ - numactl \ numactl-devel \ openssl-devel \ pandoc \ patch \ patchelf \ - pciutils \ pciutils-devel \ protobuf-c-devel \ python${PYTHON_VERSION}-devel \ python${PYTHON_VERSION}-pip \ rpm-build \ - sg3_utils \ - squashfs-tools \ sudo \ valgrind-devel \ which \ ncurses-devel \ yasm +if [[ "${INSTALL_BUILD_CI_ONLY:-}" != "true" ]]; then + # Optional packages for full-featured images; can be skipped in essential-only mode. + # shellcheck disable=SC2086 + dnf --nodocs install ${dnf_install_args} \ + clang \ + clang-tools-extra \ + fuse3 \ + gperftools-devel \ + graphviz \ + ipmctl \ + java-1.8.0-openjdk \ + ndctl \ + numactl \ + sg3_utils \ + squashfs-tools +fi + if [[ -z "${NO_OPENMPI_DEVEL+set}" ]]; then # shellcheck disable=SC2086 dnf --nodocs install ${dnf_install_args} \ diff --git a/utils/scripts/install-leap15.sh b/utils/scripts/install-leap15.sh index 3d671642820..633929b8d43 100755 --- a/utils/scripts/install-leap15.sh +++ b/utils/scripts/install-leap15.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # (C) Copyright 2025 Google LLC +# Copyright 2026 Hewlett Packard Enterprise Development LP # Install OS updates and package. Include basic tools and daos dependencies # that come from the core repo. @@ -21,25 +22,18 @@ dnf_install_args="${1:-}" dnf --nodocs install ${dnf_install_args} \ boost-devel \ bzip2 \ - curl \ - clang \ cmake \ createrepo_c \ cunit-devel \ fdupes \ flex \ - fuse3 \ gcc \ gcc-c++ \ git \ go \ go-race \ - gperftools-devel \ - graphviz \ - gzip \ hdf5-devel \ hwloc-devel \ - java-1_8_0-openjdk-devel \ libaio-devel \ libasan8 \ libcmocka-devel \ @@ -48,7 +42,6 @@ dnf --nodocs install ${dnf_install_args} \ libibverbs-devel \ libiscsi-devel \ libjson-c-devel \ - libltdl7 \ liblz4-devel \ libndctl-devel \ libnl3-devel \ @@ -63,31 +56,35 @@ dnf --nodocs install ${dnf_install_args} \ libyaml-devel \ lua-lmod \ make \ - maven \ nasm \ - numactl \ openmpi3-devel \ pandoc \ patch \ patchelf \ - pciutils \ pciutils-devel \ python${PYTHON_VERSION//./}-devel \ rpm-build \ scons \ - sg3_utils \ sudo \ valgrind-devel \ which \ yasm -# shellcheck disable=SC2086 -dnf install ${dnf_install_args} ruby-devel -gem install json -v 2.7.6 -gem install dotenv -v 2.8.1 -gem install fpm -v 1.16.0 -if [ ! -f /usr/bin/fpm ]; then - ln -s "$(basename "$(ls -1 /usr/bin/fpm.ruby*)")" /usr/bin/fpm +if [[ "${INSTALL_BUILD_CI_ONLY:-}" != "true" ]]; then + # Optional packages for full-featured images; can be skipped in essential-only mode. + # shellcheck disable=SC2086 + dnf --nodocs install ${dnf_install_args} \ + clang \ + fuse3 \ + gperftools-devel \ + graphviz \ + gzip \ + ipmctl \ + java-1_8_0-openjdk-devel \ + maven \ + ndctl \ + numactl \ + sg3_utils fi # ipmctl is only available on x86_64 @@ -96,3 +93,12 @@ if [ "$arch" = x86_64 ]; then dnf --nodocs install ${dnf_install_args} \ ipmctl-devel fi +# shellcheck disable=SC2086 +dnf --nodocs install ${dnf_install_args} ruby-devel +gem install json -v 2.7.6 +gem install dotenv -v 2.8.1 +gem install fpm -v 1.16.0 +if [ ! -f /usr/bin/fpm ]; then + ln -s "$(basename "$(ls -1 /usr/bin/fpm.ruby*)")" /usr/bin/fpm +fi +