From 0b8165c3b8c35dfad4d9cbe1d5cee883e7a9aad4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 14 Mar 2023 17:09:16 +0100 Subject: [PATCH 1/4] PR to test new bot version and new software-layer scripts --- EESSI-pilot-install-software.sh | 1 + eessi-2021.12.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 2830754b29..96adadc5fd 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -430,6 +430,7 @@ $EB SciPy-bundle-2021.05-foss-2021a.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" ### add packages here +$EB CaDiCaL-1.3.0-GCC-9.3.0.eb --robot echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml index 210bbb2845..25ea2f0f7a 100644 --- a/eessi-2021.12.yml +++ b/eessi-2021.12.yml @@ -67,3 +67,7 @@ software: versions: '3.9.1.1': versionsuffix: -dmpar + CaDiCaL: + toolchains: + GCC-9.3.0: + versions: ['1.3.0'] From dcc4976defc2f011c03b75e6ab87440db0351ea3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 14 Mar 2023 20:50:04 +0100 Subject: [PATCH 2/4] small script to check rate limits --- check_rate_limits.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 check_rate_limits.sh diff --git a/check_rate_limits.sh b/check_rate_limits.sh new file mode 100755 index 0000000000..acc5067015 --- /dev/null +++ b/check_rate_limits.sh @@ -0,0 +1,23 @@ +#!/usr/bin/bash +ALL=0 +if [[ $# -gt 0 ]]; then + ALL=1 +fi + +if [[ ! -z ${GITHUB_TOKEN} ]]; then + echo "checking limits for GITHUB_TOKEN" +else + echo "checking limits for unauthorized requests" +fi + +echo "now..: $(date)" + +if [[ ! -z ${GITHUB_TOKEN} ]]; then + echo "reset: $(date -d @$(curl --silent -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.reset))" + echo "left : $(curl --silent -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.remaining)" + [[ "${ALL}" -eq "1" ]] && echo "all : $(curl --silent -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq)" +else + echo "reset: $(date -d @$(curl --silent -L -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.reset))" + echo "left : $(curl --silent -L -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.remaining)" + [[ "${ALL}" -eq "1" ]] && echo "all : $(curl --silent -L -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq)" +fi From 51100034912a1bd92e520ac778b660f29d08ba70 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 14 Mar 2023 20:54:55 +0100 Subject: [PATCH 3/4] use check_rate_limits.sh --- EESSI-pilot-install-software.sh | 7 +++++++ bot/build.sh | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 96adadc5fd..80ccf37190 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -206,6 +206,9 @@ fi echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." +echo "check rate limits before first -[-include-easyblock]-from-pr" +./check_rate_limits.sh ALL + # install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, # see https://github.com/EESSI/software-layer/issues/123 # and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 @@ -415,6 +418,10 @@ $EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot # use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 # which includes a patch to fix installation on POWER $EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + +echo "check rate limits after last -[-include-easyblock]-from-pr" +./check_rate_limits.sh ALL + # ignore failing FlexiBLAS tests when building on POWER; # some tests are failing due to a segmentation fault due to "invalid memory reference", # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; diff --git a/bot/build.sh b/bot/build.sh index 20334501ed..d5c495086e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -156,12 +156,18 @@ fi # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) +echo "check rate limits before launching build step" +./check_rate_limits.sh ALL + echo "Executing command to build software:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} +echo "check rate limits after running build step" +./check_rate_limits.sh ALL + # prepare directory to store tarball of tmp for tarball step TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR} From c2b43e34cc391d7d3191d5ba137ad308bda8eba5 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 15 Mar 2023 07:04:02 +0100 Subject: [PATCH 4/4] make sure that CPU arch specific directories are used as container cache dir --- bot/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index d5c495086e..a9ca0bd7d6 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -27,6 +27,7 @@ source scripts/cfg_files.sh # defaults export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +HOST_ARCH=$(uname -m) # check if ${JOB_CFG_FILE} exists if [[ ! -r "${JOB_CFG_FILE}" ]]; then @@ -55,6 +56,8 @@ echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + # make sure that separate directories are used for different CPU families + SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH} export SINGULARITY_CACHEDIR fi