diff --git a/.github/workflows/push_images.yml b/.github/workflows/push_images.yml index 18e5d2e..db01a9c 100644 --- a/.github/workflows/push_images.yml +++ b/.github/workflows/push_images.yml @@ -165,6 +165,76 @@ jobs: container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" run: docker push ${container} + build_treescape_container: + needs: + - setup_shared_matrix + - build_thicket_container + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + tag: + - ${{ github.event.inputs.tag }} + tutorial_dir: + - ${{ github.event.inputs.tutorial_dir }} + docker_os: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_os }} + docker_arch: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_arch }} + containers_to_build: + - ["docker/Dockerfile.treescape", "ghcr.io/llnl/treescape"] + + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Remove unneeded stuff to make space for container + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: false + swap-storage: true + + - name: Set up Docker + uses: docker/setup-docker-action@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull layers if they exist + env: + container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" + run: docker pull ${container} || echo "${container} has not yet been pushed" + + - name: Build container + env: + context: ${{ matrix.tutorial_dir }} + dockerfile: ${{ matrix.containers_to_build[0] }} + container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" + platform: "${{ matrix.docker_os }}/${{ matrix.docker_arch }}" + run: | + cd ${context} + docker build --progress=plain --platform ${platform} -f ${dockerfile} -t ${container} . + + - name: Deploy container + env: + container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" + run: docker push ${container} + build_benchpark_container: needs: - setup_shared_matrix diff --git a/.github/workflows/test_build_images.yml b/.github/workflows/test_build_images.yml index 184b53c..e170314 100644 --- a/.github/workflows/test_build_images.yml +++ b/.github/workflows/test_build_images.yml @@ -1,15 +1,15 @@ -name: Build containers for testing PRs +name: Build containers for testing PRs on: pull_request: branches: [ "develop" ] - + jobs: setup_shared_matrix: runs-on: ubuntu-latest outputs: matrix_data: ${{ steps.set-matrix.outputs.matrix_data }} - + steps: - uses: actions/checkout@v4 with: @@ -35,7 +35,7 @@ jobs: docker_arch: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_arch }} containers_to_build: - ["docker/Dockerfile.caliper", "ghcr.io/llnl/caliper"] - + steps: - uses: actions/checkout@v4 with: @@ -44,29 +44,29 @@ jobs: - name: Remove unneeded stuff to make space for container uses: jlumbroso/free-disk-space@v1.3.1 with: - tool-cache: true + tool-cache: true android: true dotnet: true haskell: true large-packages: true - docker-images: false + docker-images: false swap-storage: true - + - name: Set up Docker uses: docker/setup-docker-action@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Pull layers if they exist env: container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" run: docker pull ${container} || echo "${container} has not yet been pushed" - - - name: Build container + + - name: Build container env: context: ${{ matrix.tutorial_dir }} dockerfile: ${{ matrix.containers_to_build[0] }} @@ -75,19 +75,19 @@ jobs: run: | cd ${context} docker build --progress=plain --platform ${platform} -f ${dockerfile} -t ${container} . - + - name: Save container to tarball env: container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" tar_dest: "${{ runner.temp }}/caliper_${{ matrix.docker_arch }}.tar" run: docker save -o ${tar_dest} ${container} - + - name: Upload tarball as artifact to be used in downstream jobs uses: actions/upload-artifact@v4 with: name: caliper_image_${{ matrix.docker_arch }} path: ${{ runner.temp }}/caliper_${{ matrix.docker_arch }}.tar - + build_thicket_container: needs: - setup_shared_matrix @@ -103,7 +103,7 @@ jobs: docker_arch: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_arch }} containers_to_build: - ["docker/Dockerfile.thicket", "ghcr.io/llnl/thicket"] - + steps: - uses: actions/checkout@v4 with: @@ -112,29 +112,29 @@ jobs: - name: Remove unneeded stuff to make space for container uses: jlumbroso/free-disk-space@v1.3.1 with: - tool-cache: true + tool-cache: true android: true dotnet: true haskell: true large-packages: true - docker-images: false + docker-images: false swap-storage: true - + - name: Set up Docker uses: docker/setup-docker-action@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Download Caliper image uses: actions/download-artifact@v4 with: name: caliper_image_${{ matrix.docker_arch }} path: ${{ runner.temp }} - + - name: Load Caliper image env: DEPENDENCY_IMAGE_TAR: ${{ runner.temp }}/caliper_${{ matrix.docker_arch }}.tar @@ -145,8 +145,8 @@ jobs: env: container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" run: docker pull ${container} || echo "${container} has not yet been pushed" - - - name: Build container + + - name: Build container env: context: ${{ matrix.tutorial_dir }} dockerfile: ${{ matrix.containers_to_build[0] }} @@ -155,20 +155,20 @@ jobs: run: | cd ${context} docker build --progress=plain --platform ${platform} -f ${dockerfile} -t ${container} . - + - name: Save container to tarball env: container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" tar_dest: "${{ runner.temp }}/thicket_${{ matrix.docker_arch }}.tar" run: docker save -o ${tar_dest} ${container} - + - name: Upload tarball as artifact to be used in downstream jobs uses: actions/upload-artifact@v4 with: name: thicket_image_${{ matrix.docker_arch }} path: ${{ runner.temp }}/thicket_${{ matrix.docker_arch }}.tar - - build_benchpark_container: + + build_treescape_container: needs: - setup_shared_matrix - build_thicket_container @@ -182,8 +182,8 @@ jobs: docker_os: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_os }} docker_arch: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_arch }} containers_to_build: - - ["docker/Dockerfile.benchpark", "ghcr.io/llnl/benchpark"] - + - ["docker/Dockerfile.treescape", "ghcr.io/llnl/treescape"] + steps: - uses: actions/checkout@v4 with: @@ -192,29 +192,29 @@ jobs: - name: Remove unneeded stuff to make space for container uses: jlumbroso/free-disk-space@v1.3.1 with: - tool-cache: true + tool-cache: true android: true dotnet: true haskell: true large-packages: true - docker-images: false + docker-images: false swap-storage: true - + - name: Set up Docker uses: docker/setup-docker-action@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Download Thicket image uses: actions/download-artifact@v4 with: name: thicket_image_${{ matrix.docker_arch }} path: ${{ runner.temp }} - + - name: Load Thicket image env: DEPENDENCY_IMAGE_TAR: ${{ runner.temp }}/thicket_${{ matrix.docker_arch }}.tar @@ -225,8 +225,8 @@ jobs: env: container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" run: docker pull ${container} || echo "${container} has not yet been pushed" - - - name: Build container + + - name: Build container env: context: ${{ matrix.tutorial_dir }} dockerfile: ${{ matrix.containers_to_build[0] }} @@ -235,19 +235,99 @@ jobs: run: | cd ${context} docker build --progress=plain --platform ${platform} -f ${dockerfile} -t ${container} . - + + - name: Save container to tarball + env: + container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" + tar_dest: "${{ runner.temp }}/treescape_${{ matrix.docker_arch }}.tar" + run: docker save -o ${tar_dest} ${container} + + - name: Upload tarball as artifact to be used in downstream jobs + uses: actions/upload-artifact@v4 + with: + name: treescape_image_${{ matrix.docker_arch }} + path: ${{ runner.temp }}/treescape_${{ matrix.docker_arch }}.tar + + build_benchpark_container: + needs: + - setup_shared_matrix + - build_treescape_container + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + tag: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).tag }} + tutorial_dir: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).tutorial_dir }} + docker_os: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_os }} + docker_arch: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_arch }} + containers_to_build: + - ["docker/Dockerfile.benchpark", "ghcr.io/llnl/benchpark"] + + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Remove unneeded stuff to make space for container + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: false + swap-storage: true + + - name: Set up Docker + uses: docker/setup-docker-action@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Download Treescape image + uses: actions/download-artifact@v4 + with: + name: treescape_image_${{ matrix.docker_arch }} + path: ${{ runner.temp }} + + - name: Load Treescape Thicket image + env: + DEPENDENCY_IMAGE_TAR: ${{ runner.temp }}/treescape_${{ matrix.docker_arch }}.tar + run: | + docker load --input ${DEPENDENCY_IMAGE_TAR} + docker image ls -a + - name: Pull layers if they exist + env: + container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" + run: docker pull ${container} || echo "${container} has not yet been pushed" + + - name: Build container + env: + context: ${{ matrix.tutorial_dir }} + dockerfile: ${{ matrix.containers_to_build[0] }} + container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" + platform: "${{ matrix.docker_os }}/${{ matrix.docker_arch }}" + run: | + cd ${context} + docker build --progress=plain --platform ${platform} -f ${dockerfile} -t ${container} . + - name: Save container to tarball env: container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" tar_dest: "${{ runner.temp }}/benchpark_${{ matrix.docker_arch }}.tar" run: docker save -o ${tar_dest} ${container} - + - name: Upload tarball as artifact to be used in downstream jobs uses: actions/upload-artifact@v4 with: name: benchpark_image_${{ matrix.docker_arch }} path: ${{ runner.temp }}/benchpark_${{ matrix.docker_arch }}.tar - + build_spawn_container: needs: - setup_shared_matrix @@ -263,7 +343,7 @@ jobs: docker_arch: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_arch }} containers_to_build: - ["docker/Dockerfile.spawn", "ghcr.io/llnl/reproducible-benchmarking-spawn"] - + steps: - uses: actions/checkout@v4 with: @@ -272,29 +352,29 @@ jobs: - name: Remove unneeded stuff to make space for container uses: jlumbroso/free-disk-space@v1.3.1 with: - tool-cache: true + tool-cache: true android: true dotnet: true haskell: true large-packages: true - docker-images: false + docker-images: false swap-storage: true - + - name: Set up Docker uses: docker/setup-docker-action@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Download Benchpark image uses: actions/download-artifact@v4 with: name: benchpark_image_${{ matrix.docker_arch }} path: ${{ runner.temp }} - + - name: Load Benchpark image env: DEPENDENCY_IMAGE_TAR: ${{ runner.temp }}/benchpark_${{ matrix.docker_arch }}.tar @@ -305,8 +385,8 @@ jobs: env: container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" run: docker pull ${container} || echo "${container} has not yet been pushed" - - - name: Build container + + - name: Build container env: context: ${{ matrix.tutorial_dir }} dockerfile: ${{ matrix.containers_to_build[0] }} @@ -315,7 +395,7 @@ jobs: run: | cd ${context} docker build --progress=plain --platform ${platform} -f ${dockerfile} -t ${container} . - + build_init_container: needs: - setup_shared_matrix @@ -330,7 +410,7 @@ jobs: docker_arch: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_arch }} containers_to_build: - ["docker/Dockerfile.init", "ghcr.io/llnl/reproducible-benchmarking-init"] - + steps: - uses: actions/checkout@v4 with: @@ -339,17 +419,17 @@ jobs: - name: Remove unneeded stuff to make space for container uses: jlumbroso/free-disk-space@v1.3.1 with: - tool-cache: true + tool-cache: true android: true dotnet: true haskell: true large-packages: true - docker-images: false + docker-images: false swap-storage: true - + - name: Set up Docker uses: docker/setup-docker-action@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -360,8 +440,8 @@ jobs: env: container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" run: docker pull ${container} || echo "${container} has not yet been pushed" - - - name: Build container + + - name: Build container env: context: ${{ matrix.tutorial_dir }} dockerfile: ${{ matrix.containers_to_build[0] }} @@ -370,7 +450,7 @@ jobs: run: | cd ${context} docker build --progress=plain --platform ${platform} -f ${dockerfile} -t ${container} . - + build_hub_container: needs: - setup_shared_matrix @@ -385,7 +465,7 @@ jobs: docker_arch: ${{ fromJson(needs.setup_shared_matrix.outputs.matrix_data).docker_arch }} containers_to_build: - ["docker/Dockerfile.hub", "ghcr.io/llnl/reproducible-benchmarking-hub"] - + steps: - uses: actions/checkout@v4 with: @@ -394,17 +474,17 @@ jobs: - name: Remove unneeded stuff to make space for container uses: jlumbroso/free-disk-space@v1.3.1 with: - tool-cache: true + tool-cache: true android: true dotnet: true haskell: true large-packages: true - docker-images: false + docker-images: false swap-storage: true - + - name: Set up Docker uses: docker/setup-docker-action@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -415,8 +495,8 @@ jobs: env: container: "${{ matrix.containers_to_build[1] }}:${{ matrix.tag }}" run: docker pull ${container} || echo "${container} has not yet been pushed" - - - name: Build container + + - name: Build container env: context: ${{ matrix.tutorial_dir }} dockerfile: ${{ matrix.containers_to_build[0] }} diff --git a/.gitmodules b/.gitmodules index b28c69a..640f9bc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -28,3 +28,6 @@ [submodule "2026-RIKEN/tutorial-code/caliper-tutorial"] path = 2026-RIKEN/tutorial-code/caliper-tutorial url = https://github.com/daboehme/caliper-tutorial.git +[submodule "latest/tutorial-code/treescape-media"] + path = latest/tutorial-code/treescape-media + url = https://github.com/llnl/treescape-media diff --git a/github_ci_matrix.json b/github_ci_matrix.json index ad3f5d5..6180457 100644 --- a/github_ci_matrix.json +++ b/github_ci_matrix.json @@ -1,9 +1,9 @@ { "tag": [ - "riken-2026" + "latest" ], "tutorial_dir": [ - "2026-RIKEN" + "latest" ], "docker_os": [ "linux" diff --git a/latest/docker/Dockerfile.benchpark b/latest/docker/Dockerfile.benchpark index e92034e..b78235b 100644 --- a/latest/docker/Dockerfile.benchpark +++ b/latest/docker/Dockerfile.benchpark @@ -4,9 +4,9 @@ # SPDX-License-Identifier: Apache-2.0 # For testing -# FROM test-thicket +# FROM test-treescape -FROM ghcr.io/llnl/thicket:latest +FROM ghcr.io/llnl/treescape:latest USER root diff --git a/latest/docker/Dockerfile.spawn b/latest/docker/Dockerfile.spawn index 398d038..93ff179 100644 --- a/latest/docker/Dockerfile.spawn +++ b/latest/docker/Dockerfile.spawn @@ -84,12 +84,17 @@ RUN cmake \ COPY ./tutorial-code/caliper-tutorial/tutorial ${HOME}/caliper-tutorial/ COPY ./tutorial-code/caliper-tutorial/apps ${HOME}/caliper-tutorial/apps + COPY ./tutorial-code/thicket-tutorial/data/lassen ${HOME}/thicket-tutorial/data/lassen COPY ./tutorial-code/thicket-tutorial/data/quartz ${HOME}/thicket-tutorial/data/quartz COPY ./tutorial-code/thicket-tutorial/notebooks/01_thicket_tutorial.ipynb ${HOME}/thicket-tutorial/notebooks/01_thicket_tutorial.ipynb COPY ./tutorial-code/thicket-tutorial/notebooks/02_thicket_rajaperf_clustering.ipynb ${HOME}/thicket-tutorial/notebooks/02_thicket_rajaperf_clustering.ipynb COPY ./tutorial-code/thicket-tutorial/LICENSE ${HOME}/thicket-tutorial +COPY ./tutorial-code/treescape-media/datasets/newdemo/test ${HOME}/treescape-media/data/newdemo/test +COPY ./tutorial-code/treescape-media/examples/NightlyTestDemo_local.ipynb ${HOME}/treescape-media/examples/NightlyTestDemo_local.ipynb +COPY ./tutorial-code/treescape-media/LICENSE ${HOME}/treescape-media + COPY tutorial-code/system-description/aws-tutorial ${HOME}/benchpark/systems/aws-tutorial COPY tutorial-code/system-description/AWS_Tutorial-c7i-EFA ${HOME}/benchpark/systems/all_hardware_descriptions/AWS_Tutorial-c7i-EFA diff --git a/latest/docker/Dockerfile.treescape b/latest/docker/Dockerfile.treescape new file mode 100644 index 0000000..f2f8243 --- /dev/null +++ b/latest/docker/Dockerfile.treescape @@ -0,0 +1,21 @@ +# Copyright 2025 Lawrence Livermore National Security, LLC and other +# Benchpark developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +# For testing +# FROM test-thicket + +FROM ghcr.io/llnl/thicket:latest + +USER root + +RUN git clone https://github.com/LLNL/treescape ${GIT_CLONE_STAGING_DIR}/treescape && \ + cd ${GIT_CLONE_STAGING_DIR}/treescape && \ + git checkout main-2026-04-13 + +RUN . /opt/global_py_venv/bin/activate && \ + cd ${GIT_CLONE_STAGING_DIR}/treescape && \ + python3 -m pip install . + +USER ${NB_USER} diff --git a/latest/docker/build_and_upload_images.sh b/latest/docker/build_and_upload_images.sh index db65beb..0b0b932 100755 --- a/latest/docker/build_and_upload_images.sh +++ b/latest/docker/build_and_upload_images.sh @@ -20,7 +20,7 @@ TAG="$1" DOCKER_PLATFORMS="linux/amd64,linux/arm64" -TO_BUILD_IDS=( "caliper" "thicket" "benchpark" "init" "spawn" ) +TO_BUILD_IDS=( "caliper" "thicket" "treescape" "benchpark" "init" "spawn" ) if [ $# -ge 2 ]; then TO_BUILD_IDS=( "$2" ) @@ -28,6 +28,7 @@ fi caliper_IMAGE="ghcr.io/ilumsden/latest-caliper" thicket_IMAGE="ghcr.io/ilumsden/latest-thicket" +treescape_IMAGE="ghcr.io/ilumsden/latest-treescape" benchpark_IMAGE="ghcr.io/ilumsden/latest-benchpark" init_IMAGE="ghcr.io/ilumsden/latest-test-init" spawn_IMAGE="ghcr.io/ilumsden/latest-test-spawn" diff --git a/latest/tutorial-code/treescape-media b/latest/tutorial-code/treescape-media new file mode 160000 index 0000000..2b01939 --- /dev/null +++ b/latest/tutorial-code/treescape-media @@ -0,0 +1 @@ +Subproject commit 2b0193916bbdf8123222ee909c313323790a8ef8