Test bot PR155 with software-layer PR237

.github/workflows/test_eessi.yml

@@ -40,4 +40,33 @@ jobs:
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
+              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
+
+        - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
+          run: |
+              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+              module load EasyBuild
+              eb --version
+              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              env | grep ^EESSI | sort
+              echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)"
+              echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "    toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
+              # note, check_missing_installations.sh exits 1 if a package was
+              # missing, which is intepreted as false (exit code based, not
+              # boolean logic), hence when the script exits 0 if no package was
+              # missing it is interpreted as true, thus the test did not capture
+              # the missing package
+              if ./check_missing_installations.sh; then
+                echo "did NOT capture missing package; test FAILED"
+                exit 1
+              else
+                echo "captured missing package; test PASSED"
+                exit 0
+              fi

EESSI-pilot-install-software.sh

@@ -206,6 +206,9 @@ fi
 
 echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..."
 
+echo "check rate limits before first -[-include-easyblock]-from-pr"
+./check_rate_limits.sh ALL
+
 # install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up,
 # see https://github.com/EESSI/software-layer/issues/123
 # and https://github.com/easybuilders/easybuild-easyblocks/pull/2557
@@ -415,6 +418,10 @@ $EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot
 # use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885
 # which includes a patch to fix installation on POWER
 $EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot
+
+echo "check rate limits after last -[-include-easyblock]-from-pr"
+./check_rate_limits.sh ALL
+
 # ignore failing FlexiBLAS tests when building on POWER;
 # some tests are failing due to a segmentation fault due to "invalid memory reference",
 # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476;
@@ -430,6 +437,7 @@ $EB SciPy-bundle-2021.05-foss-2021a.eb --robot
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
 ### add packages here
+$EB CaDiCaL-1.3.0-GCC-9.3.0.eb --robot
 
 echo ">> Creating/updating Lmod cache..."
 export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua"

bot/build.sh

@@ -27,6 +27,7 @@ source scripts/cfg_files.sh
 
 # defaults
 export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+HOST_ARCH=$(uname -m)
 
 # check if ${JOB_CFG_FILE} exists
 if [[ ! -r "${JOB_CFG_FILE}" ]]; then
@@ -55,6 +56,8 @@ echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'"
 SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
 echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
+    # make sure that separate directories are used for different CPU families
+    SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH}
     export SINGULARITY_CACHEDIR
 fi
 
@@ -156,12 +159,18 @@ fi
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
 
+echo "check rate limits before launching build step"
+./check_rate_limits.sh ALL
+
 echo "Executing command to build software:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}"
 echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 ./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \
                      -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
+echo "check rate limits after running build step"
+./check_rate_limits.sh ALL
+
 # prepare directory to store tarball of tmp for tarball step
 TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step
 mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR}

check_missing_installations.sh

@@ -27,5 +27,21 @@ eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
 # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
 # PR #16531: Nextflow-22.10.1.eb
 ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out}
-grep "No missing modules" ${eb_missing_out} > /dev/null
-check_exit_code $? "${ok_msg}" "${fail_msg}"
+
+# the above assesses the installed software for each easyconfig provided in
+# the easystack file and then print messages such as
+# `No missing modules!`
+# or
+# `2 out of 3 required modules missing:`
+# depending on the result of the assessment. Hence, we need to check if the
+# output does not contain any line with ` required modules missing:`
+
+grep " required modules missing:" ${eb_missing_out} > /dev/null
+
+# we need to process the result (from finding `No missing modules` to NOT finding
+# ` required modules missing:` and no other error happened)
+#
+# if grep returns 1 (` required modules missing:` was NOT found), we set
+# MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1
+[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1
+check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}"

check_rate_limits.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/bash
+ALL=0
+if [[ $# -gt 0 ]]; then
+  ALL=1
+fi
+
+if [[ ! -z ${GITHUB_TOKEN} ]]; then
+  echo "checking limits for GITHUB_TOKEN"
+else
+  echo "checking limits for unauthorized requests"
+fi
+
+echo "now..: $(date)"
+
+if [[ ! -z ${GITHUB_TOKEN} ]]; then
+  echo "reset: $(date -d @$(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "Authorization: Bearer ${GITHUB_TOKEN}"  -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.reset))"
+  echo "left : $(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "Authorization: Bearer ${GITHUB_TOKEN}"  -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.remaining)"
+  [[ "${ALL}" -eq "1" ]] && echo "all  : $(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "Authorization: Bearer ${GITHUB_TOKEN}"  -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq)"
+else
+  echo "reset: $(date -d @$(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.reset))"
+  echo "left : $(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.remaining)"
+  [[ "${ALL}" -eq "1" ]] && echo "all  : $(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq)"
+fi

eessi-2021.12.yml

@@ -67,3 +67,7 @@ software:
          versions: 
            '3.9.1.1': 
              versionsuffix: -dmpar
+  CaDiCaL:
+    toolchains:
+      GCC-9.3.0:
+        versions: ['1.3.0']