comment out viewer cells

modify gitignore for service account keys
run notebooks in their own temp folder
add a step to create pull request to update colab images list

Author: vkt1414

.github/workflows/test_colab.yml

@@ -3,17 +3,24 @@ name: Check Commits and Colab Images
 on:
   push:
     branches: [ "master" ]
+    paths:
+      - ".github/workflows/test_colab.yml"
+      - "notebooks/getting_started/part**.ipynb"
   pull_request:
     branches: [ "master" ]
+    paths:
+      - ".github/workflows/test_colab.yml"
+      - "notebooks/getting_started/part**.ipynb"
   workflow_dispatch:
   schedule:
     - cron: 0 12 */1 * *
 
 jobs:
-  check_commits_and_images:
+  check_colab_images:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      pull-requests: write
 
     steps:
       - name: Checkout code
@@ -28,15 +35,15 @@ jobs:
         run: pip install requests==2.31.0 pandas==2.1.1 google-cloud-bigquery==3.12.0 pyarrow==13.0.0 nbformat==5.9.2
 
       - name: Authorize Google Cloud
+        if: github.event_name != 'pull_request'
         uses: google-github-actions/auth@v1
         with:
           credentials_json: ${{ secrets.SERVICE_ACCOUNT_KEY }}
           create_credentials_file: true
           export_environment_variables: true
 
-      - name: Run check-commits.py and check-colab-images.py, preprocess notebooks scripts
+      - name: Run check-colab-images.py and preprocess notebooks
         run: |
-          python test/src/check-commits.py
           python test/src/check-colab-images.py
           python test/src/preProcessNotebooks.py
 
@@ -60,59 +67,59 @@ jobs:
           swap-storage: true
 
       - name: Docker login
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Pull from GCP and Push Docker image to Docker Hub    
-        if: env.RESULT == 'true'
+        if: github.event_name != 'pull_request' && env.RESULT == 'true'
         run: |
           docker pull us-docker.pkg.dev/colab-images/public/runtime:latest
           docker tag us-docker.pkg.dev/colab-images/public/runtime:latest imagingdatacommons/idc-testing-colab:latest
           docker push imagingdatacommons/idc-testing-colab:latest
 
       - name: Pull Docker image from Docker Hub
-        if: env.RESULT == 'false'
+        if: github.event_name == 'pull_request' || env.RESULT == 'false'
         run: |
           docker pull imagingdatacommons/idc-testing-colab:latest
           
       - name: Copy Google Cloud credentials to Docker container
+        if: github.event_name != 'pull_request'
         run: |
           CREDENTIALS_FILE_PATH="${{ env.GOOGLE_APPLICATION_CREDENTIALS }}"
           CREDENTIALS_FILE_NAME=$(basename "$CREDENTIALS_FILE_PATH")
           GOOGLE_APPLICATION_CREDENTIALS="/content/$CREDENTIALS_FILE_NAME"
           echo "GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS" >> $GITHUB_ENV        
           
       - name: Run notebook with papermill
+        if: github.event_name != 'pull_request'
         run: |
           for nb in part1_prerequisites part2_searching_basics part3_exploring_cohorts; do
+            # Create a new directory for each notebook
+            sudo mkdir -p "temp/${nb}"
+            
             docker run -d --name colab -v "$(pwd):/content" -e GOOGLE_APPLICATION_CREDENTIALS="${{ env.GOOGLE_APPLICATION_CREDENTIALS }}" imagingdatacommons/idc-testing-colab:latest
             docker exec -t colab /bin/bash -c "pip install papermill==2.4.0"
-            docker exec -t colab /bin/bash -c "set -o xtrace && set -o errexit && set -o pipefail && set -o nounset && set +o errexit && cd content/ && papermill --log-level='DEBUG' /content/notebooks/getting_started/${nb}.ipynb /content/test/outputs/${nb}_papermill_output.ipynb && set -o errexit && ls -A"
-            #docker exec -t colab /bin/bash -c "jupyter nbconvert --to html --ExtractOutputPreprocessor.enabled=False /content/test/outputs/output_${nb}.ipynb"
+            
+            # Change the working directory to the new directory
+            docker exec -t colab /bin/bash -c "set -o xtrace && set -o errexit && set -o pipefail && set -o nounset && set +o errexit && cd /content/temp/${nb} && papermill --log-level='DEBUG' /content/notebooks/getting_started/${nb}.ipynb /content/test/outputs/${nb}_papermill_output.ipynb && set -o errexit && ls -A"
+                     
             docker stop colab
             docker rm colab
+            sudo rm -r "temp/${nb}"
           done
-
-      # skip commit
-      #- name: Commit changes
-      #  if: ${{ github.event_name != 'pull_request' }}
-      #  uses: stefanzweifel/git-auto-commit-action@v4
-      #  with:
-      #    commit_message: 'Check colab env'
-      #    file_pattern: 'test/*.csv test/outputs/*.ipynb'
-      #    branch: 'master'
-      
-      #- name: Check output notebooks for errors 
-      #  run: | 
-      #    for nb in part1_prerequisites part2_searching_basics part3_exploring_cohorts; do
-      #      if grep -q '"name": "stderr"\|"status": "failed"' test/outputs/output_${nb}.ipynb; then
-      #        echo "Error messages found in the ${nb} notebook output:"
-      #        cat test/outputs/output_${nb}.ipynb
-      #        exit 1
-      #      else
-      #        echo "No errors found in the ${nb} notebook output."
-      #      fi
-      #    done
-      #    exit $EXIT_CODE
+        
+      - name: Create Pull Request to update the colab image metadata database
+        if: github.event_name != 'pull_request'
+        uses: peter-evans/create-pull-request@v5
+        with:
+          title: Update colab images list
+          body:  Update colab-images-list.csv so that when a colab image is updated, we do not keep pulling colab docker image from gcp and pushing to dockerhub everytime
+          base:  master
+          branch: update-colab-images-list
+          add-paths: |
+            test/colab-images-list.csv
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -5,3 +5,6 @@ __pycache__/
 # Pycharm project files
 .idea
 .idea/*
+
+
+gha-creds-**.json

test/colab-images-list.csv

@@ -1,7 +1,8 @@
 date,tag,sha256,docker_pull_tag,docker_pull_sha256_tag
-,latest,sha256:4a26494c9c92ab4d0515e0715d79dfecbe8cfacb9b86fcfd55bc0274cb89530d,us-docker.pkg.dev/colab-images/public/runtime:latest,us-docker.pkg.dev/colab-images/public/runtime@sha256:4a26494c9c92ab4d0515e0715d79dfecbe8cfacb9b86fcfd55bc0274cb89530d
+,latest,sha256:9634a9786cde027a5c115a2a2df31aebe0c9ef9a4a06951cb32bcf6edec84ef7,us-docker.pkg.dev/colab-images/public/runtime:latest,us-docker.pkg.dev/colab-images/public/runtime@sha256:9634a9786cde027a5c115a2a2df31aebe0c9ef9a4a06951cb32bcf6edec84ef7
 20230515,release-colab-20230515-060150-RC00,sha256:3a8fc58f7e81b96dc59a2fb48b7973802f59fdd634fb538569228d830a7e76a9,us-docker.pkg.dev/colab-images/public/runtime:release-colab-20230515-060150-RC00,us-docker.pkg.dev/colab-images/public/runtime@sha256:3a8fc58f7e81b96dc59a2fb48b7973802f59fdd634fb538569228d830a7e76a9
 20230622,release-colab-20230622-060123-RC01,sha256:7dac57e02aae4e83aab349563190a71bdd07374e1365f53bd6a50280046c6091,us-docker.pkg.dev/colab-images/public/runtime:release-colab-20230622-060123-RC01,us-docker.pkg.dev/colab-images/public/runtime@sha256:7dac57e02aae4e83aab349563190a71bdd07374e1365f53bd6a50280046c6091
 20230711,release-colab-20230711-060203-RC00,sha256:53dc33f450cd162d8a42c5aff02d50ac24eb9fc68be77f0374614ad07247e9cd,us-docker.pkg.dev/colab-images/public/runtime:release-colab-20230711-060203-RC00,us-docker.pkg.dev/colab-images/public/runtime@sha256:53dc33f450cd162d8a42c5aff02d50ac24eb9fc68be77f0374614ad07247e9cd
 20230803,release-colab-20230803-060151-RC00,sha256:ae8a5bf22a84c67fb4b35aa4b1f19dac94b01b56a97c5c7bb15db57552e8d38c,us-docker.pkg.dev/colab-images/public/runtime:release-colab-20230803-060151-RC00,us-docker.pkg.dev/colab-images/public/runtime@sha256:ae8a5bf22a84c67fb4b35aa4b1f19dac94b01b56a97c5c7bb15db57552e8d38c
 20230921,release-colab_20230921-060057_RC00,sha256:4a26494c9c92ab4d0515e0715d79dfecbe8cfacb9b86fcfd55bc0274cb89530d,us-docker.pkg.dev/colab-images/public/runtime:release-colab_20230921-060057_RC00,us-docker.pkg.dev/colab-images/public/runtime@sha256:4a26494c9c92ab4d0515e0715d79dfecbe8cfacb9b86fcfd55bc0274cb89530d
+20231214,release-colab_20231214-060137_RC00,sha256:9634a9786cde027a5c115a2a2df31aebe0c9ef9a4a06951cb32bcf6edec84ef7,us-docker.pkg.dev/colab-images/public/runtime:release-colab_20231214-060137_RC00,us-docker.pkg.dev/colab-images/public/runtime@sha256:9634a9786cde027a5c115a2a2df31aebe0c9ef9a4a06951cb32bcf6edec84ef7

test/src/check-colab-images.py

@@ -48,7 +48,7 @@ def main():
             if existing_sha256 != current_sha256:
                 print("SHA256 digest has changed!")
                 # Perform actions if the digest has changed, like updating GitHub Actions.
-                df.to_csv('database.csv', index=False)  # Save the new DataFrame as 'database.csv'
+                df.to_csv('test/colab-images-list.csv', index=False)  # Save the new DataFrame as 'database.csv'
                 print("Updated the database CSV.")
                 result = True
             else:

test/src/preProcessNotebooks.py

@@ -1,6 +1,7 @@
 import subprocess
 import nbformat
 
+
 def modify_notebook(file_name, project_id, query1, query2):
     # Load the notebook
     with open(file_name) as f:
@@ -10,26 +11,50 @@ def modify_notebook(file_name, project_id, query1, query2):
     for cell in nb.cells:
         if cell.cell_type == "code":
             # Replace the line if it exists
-            cell.source = cell.source.replace('my_ProjectID = "" #@param {type:"string"}', f"#this project_id is injected for testing\nmy_ProjectID='{project_id}'")
+            cell.source = cell.source.replace(
+                'my_ProjectID = "" #@param {type:"string"}',
+                f"#this project_id is injected for testing\nmy_ProjectID='{project_id}'",
+            )
             # Comment out another line
-            cell.source = cell.source.replace('auth.authenticate_user()', '#while testing, the authentication is handled by using application default credentials\n#auth.authenticate_user()')
-            cell.source = cell.source.replace('REPLACE THIS TEXT WITH YOUR QUERY!', query1)
-            cell.source = cell.source.replace('# write the selection criteria under this line!', query2)            
+            cell.source = cell.source.replace(
+                "auth.authenticate_user()",
+                "#while testing, the authentication is handled by using application default credentials\n#auth.authenticate_user()",
+            )
+            cell.source = cell.source.replace(
+                "REPLACE THIS TEXT WITH YOUR QUERY!", query1
+            )
+            cell.source = cell.source.replace(
+                "# write the selection criteria under this line!", query2
+            )
+            cell.source = (
+                "# " + cell.source.replace("\n", "\n# ")
+                if cell.source.lower().startswith("viewer")
+                else cell.source
+            )
 
     # Write the notebook back to disk
-    with open(file_name, 'w') as f:
+    with open(file_name, "w") as f:
         nbformat.write(nb, f)
 
+
 # Command to run
 files = [
     "notebooks/getting_started/part1_prerequisites.ipynb",
     "notebooks/getting_started/part2_searching_basics.ipynb",
-    "notebooks/getting_started/part3_exploring_cohorts.ipynb"
+    "notebooks/getting_started/part3_exploring_cohorts.ipynb",
 ]
 
 for file_name in files:
 
-    project_id = 'idc-external-025'
-    query1 = '#this query is injected for testing\n SELECT DISTINCT collection_tumorLocation FROM `bigquery-public-data.idc_current.dicom_all` ' if 'part2' in file_name else ''
-    query2 = '#this query is injected for testing\n MODALITY=\'MR\' AND collection_tumorLocation=\'Lung\'' if 'part2' in file_name else ''
+    project_id = "idc-external-025"
+    query1 = (
+        "#this query is injected for testing\n SELECT DISTINCT collection_tumorLocation FROM `bigquery-public-data.idc_current.dicom_all` "
+        if "part2" in file_name
+        else ""
+    )
+    query2 = (
+        "#this query is injected for testing\n MODALITY='MR' AND collection_tumorLocation='Lung'"
+        if "part2" in file_name
+        else ""
+    )
     modify_notebook(file_name, project_id, query1, query2)