Deploy Python Package to AWS #19
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Python Package to AWS | |
| on: | |
| push: | |
| tags: | |
| # e.g. v1.0.0, v1.0.1, v1.0.0-rc1, etc. | |
| - "v[0-9]+.[0-9]+.[0-9]+*" | |
| workflow_dispatch: | |
| env: | |
| PACKAGE_NAME: layerlens | |
| jobs: | |
| validate: | |
| runs-on: ubuntu-latest | |
| environment: production | |
| outputs: | |
| release_tag: ${{ steps.set_release_tag.outputs.release_tag }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch all history for checking branch | |
| - name: Set release tag | |
| id: set_release_tag | |
| # ensure the tag is valid (matches code, is on release branch, etc) | |
| run: | | |
| RELEASE_TAG=${GITHUB_REF#refs/tags/} | |
| echo "Using tag: $RELEASE_TAG" | |
| chmod +x ./scripts/validate_release_tag.sh | |
| ./scripts/validate_release_tag.sh "$RELEASE_TAG" | |
| echo "RELEASE_TAG=$RELEASE_TAG" >> $GITHUB_ENV | |
| echo "release_tag=$RELEASE_TAG" >> $GITHUB_OUTPUT | |
| deploy: | |
| needs: validate | |
| runs-on: ubuntu-latest | |
| environment: production | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| - name: Install build dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install build twine setuptools wheel | |
| - name: Build package | |
| run: | | |
| python -m build | |
| - name: Get package version | |
| id: get_version | |
| run: | | |
| echo "Current working directory:" | |
| pwd | |
| echo "Listing files:" | |
| ls -la | |
| echo "Checking if version file exists..." | |
| if [ -f "src/atlas/_version.py" ]; then | |
| echo "Version file found:" | |
| cat src/atlas/_version.py | |
| # Extract version directly | |
| VERSION=$(grep -E '^__version__\s*=' src/atlas/_version.py | grep -o '".*"' | tr -d '"') | |
| echo "Direct extraction result: '$VERSION'" | |
| else | |
| echo "Version file not found, trying script method..." | |
| fi | |
| # Fallback to script method if direct extraction failed | |
| if [ -z "$VERSION" ]; then | |
| echo "Trying script method..." | |
| chmod +x ./scripts/get_version.sh | |
| ls -la ./scripts/get_version.sh | |
| VERSION=$(./scripts/get_version.sh 2>&1) | |
| echo "Script output: '$VERSION'" | |
| # Extract just the version (last line) in case there's debug output | |
| VERSION=$(echo "$VERSION" | tail -n 1) | |
| fi | |
| if [ -z "$VERSION" ]; then | |
| echo "Error: Could not extract version" | |
| exit 1 | |
| fi | |
| echo "Final version: '$VERSION'" | |
| echo "VERSION=$VERSION" >> $GITHUB_OUTPUT | |
| echo "Package version: $VERSION" | |
| - name: Set up AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Create package directory structure | |
| run: | | |
| mkdir -p upload/package/${{ env.PACKAGE_NAME }} | |
| mkdir -p upload/versions/${{ steps.get_version.outputs.VERSION }} | |
| - name: Copy package files to version-specific folder | |
| run: | | |
| cp dist/* upload/versions/${{ steps.get_version.outputs.VERSION }}/ | |
| - name: List existing versions from S3 | |
| id: list_versions | |
| run: | | |
| # Get existing versions from S3 (ignore errors if bucket is empty) | |
| aws s3 ls s3://${{ secrets.S3_BUCKET_NAME }}/versions/ 2>/dev/null > existing_versions.txt || echo "" > existing_versions.txt | |
| # Extract version numbers from S3 listing | |
| EXISTING_VERSIONS="" | |
| if [ -s existing_versions.txt ]; then | |
| EXISTING_VERSIONS=$(awk '{print $2}' existing_versions.txt | sed 's/\///g' | grep -v '^$' | sort -V || true) | |
| fi | |
| CURRENT_VERSION="${{ steps.get_version.outputs.VERSION }}" | |
| echo "Current version: $CURRENT_VERSION" | |
| echo "Existing versions from S3: $EXISTING_VERSIONS" | |
| # Validate current version is not empty | |
| if [ -z "$CURRENT_VERSION" ]; then | |
| echo "Error: Current version is empty" | |
| exit 1 | |
| fi | |
| # Combine existing and current versions, remove duplicates, and sort | |
| if [ -z "$EXISTING_VERSIONS" ]; then | |
| ALL_VERSIONS="$CURRENT_VERSION" | |
| else | |
| ALL_VERSIONS=$(echo -e "$EXISTING_VERSIONS\n$CURRENT_VERSION" | sort -V | uniq | grep -v '^$' || echo "$CURRENT_VERSION") | |
| fi | |
| echo "Available versions:" | |
| echo "$ALL_VERSIONS" | |
| # Save for use in HTML generation (ensure no empty lines) | |
| if [ -n "$ALL_VERSIONS" ]; then | |
| echo "$ALL_VERSIONS" | grep -v '^$' > all_versions.txt || echo "$ALL_VERSIONS" > all_versions.txt | |
| else | |
| echo "$CURRENT_VERSION" > all_versions.txt | |
| fi | |
| # Format for JSON array (for potential future use) - simplified approach | |
| VERSIONS_JSON="[" | |
| FIRST=true | |
| while IFS= read -r version; do | |
| if [ -n "$version" ]; then | |
| if [ "$FIRST" = true ]; then | |
| VERSIONS_JSON="$VERSIONS_JSON\"$version\"" | |
| FIRST=false | |
| else | |
| VERSIONS_JSON="$VERSIONS_JSON,\"$version\"" | |
| fi | |
| fi | |
| done < all_versions.txt | |
| VERSIONS_JSON="$VERSIONS_JSON]" | |
| echo "VERSIONS_JSON=$VERSIONS_JSON" >> $GITHUB_OUTPUT | |
| echo "Generated JSON: $VERSIONS_JSON" | |
| - name: Download existing versions and prepare directories | |
| run: | | |
| # Copy current version to package index | |
| cp dist/* upload/package/${{ env.PACKAGE_NAME }}/ | |
| # Download existing versions from S3 for both package index and version directories | |
| while IFS= read -r version; do | |
| if [ ! -z "$version" ] && [ "$version" != "${{ steps.get_version.outputs.VERSION }}" ]; then | |
| echo "Downloading version $version" | |
| # Create directory for this version | |
| mkdir -p upload/versions/$version | |
| # Download to both locations: | |
| # 1. To package index (for pip compatibility) | |
| aws s3 sync s3://${{ secrets.S3_BUCKET_NAME }}/versions/$version/ upload/package/${{ env.PACKAGE_NAME }}/ || echo "Version $version not found in S3" | |
| # 2. To version-specific directory (for HTML generation) | |
| aws s3 sync s3://${{ secrets.S3_BUCKET_NAME }}/versions/$version/ upload/versions/$version/ || echo "Version $version not found in S3" | |
| fi | |
| done < all_versions.txt | |
| - name: Generate package index HTML | |
| run: | | |
| cat > upload/package/${{ env.PACKAGE_NAME }}/index.html << EOF | |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <title>Links for ${{ env.PACKAGE_NAME }}</title> | |
| </head> | |
| <body> | |
| <h1>Links for ${{ env.PACKAGE_NAME }}</h1> | |
| <p><a href="../../versions.html">View all versions</a></p> | |
| EOF | |
| for file in upload/package/${{ env.PACKAGE_NAME }}/*.{tar.gz,whl}; do | |
| if [ -f "$file" ]; then | |
| filename=$(basename "$file") | |
| echo " <a href=\"$filename\">$filename</a><br/>" >> upload/package/${{ env.PACKAGE_NAME }}/index.html | |
| fi | |
| done | |
| echo "</body></html>" >> upload/package/${{ env.PACKAGE_NAME }}/index.html | |
| - name: Generate versions page HTML | |
| run: | | |
| cat > upload/versions.html << EOF | |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <title>All Versions of ${{ env.PACKAGE_NAME }}</title> | |
| <style> | |
| body { font-family: Arial, sans-serif; margin: 40px; } | |
| .version { margin: 20px 0; padding: 15px; border: 1px solid #ddd; border-radius: 5px; } | |
| .version h3 { margin-top: 0; color: #333; } | |
| .files { margin-left: 20px; } | |
| .current { background-color: #e8f5e8; border-color: #4caf50; } | |
| .install-cmd { background-color: #f5f5f5; padding: 10px; border-radius: 3px; font-family: monospace; } | |
| </style> | |
| </head> | |
| <body> | |
| <h1>All Versions of ${{ env.PACKAGE_NAME }}</h1> | |
| <p>Available versions of the ${{ env.PACKAGE_NAME }} package.</p> | |
| <div class="install-cmd"> | |
| <strong>Install latest:</strong> pip install ${{ env.PACKAGE_NAME }} --index-url https://${{ secrets.CLOUDFRONT_DOMAIN }}/package/<br/> | |
| <strong>Install specific version:</strong> pip install ${{ env.PACKAGE_NAME }}==VERSION --index-url https://${{ secrets.CLOUDFRONT_DOMAIN }}/package/ | |
| </div> | |
| EOF | |
| # Generate version listings | |
| while IFS= read -r version; do | |
| if [ ! -z "$version" ]; then | |
| echo "Processing version: $version" | |
| # Check if this is the current version | |
| CURRENT_CLASS="" | |
| if [ "$version" = "${{ steps.get_version.outputs.VERSION }}" ]; then | |
| CURRENT_CLASS=" current" | |
| fi | |
| cat >> upload/versions.html << EOF | |
| <div class="version$CURRENT_CLASS"> | |
| <h3>Version $version</h3> | |
| EOF | |
| if [ "$version" = "${{ steps.get_version.outputs.VERSION }}" ]; then | |
| echo " <p><strong>(Current/Latest Version)</strong></p>" >> upload/versions.html | |
| fi | |
| echo " <div class=\"files\">" >> upload/versions.html | |
| echo " <strong>Available files:</strong><br/>" >> upload/versions.html | |
| # List files for this version | |
| echo "Checking for files in upload/versions/$version/" | |
| ls -la upload/versions/$version/ || echo "Directory not found or empty" | |
| FOUND_FILES=false | |
| for file in upload/versions/$version/*.{tar.gz,whl}; do | |
| if [ -f "$file" ]; then | |
| filename=$(basename "$file") | |
| echo " <a href=\"versions/$version/$filename\">$filename</a><br/>" >> upload/versions.html | |
| echo "Found file: $filename" | |
| FOUND_FILES=true | |
| fi | |
| done | |
| if [ "$FOUND_FILES" = false ]; then | |
| echo " <p>No package files found for this version</p>" >> upload/versions.html | |
| echo "Warning: No files found for version $version" | |
| fi | |
| echo " </div>" >> upload/versions.html | |
| echo "</div>" >> upload/versions.html | |
| fi | |
| done < all_versions.txt | |
| echo "</body></html>" >> upload/versions.html | |
| - name: Generate main package index | |
| run: | | |
| cat > upload/package/index.html << EOF | |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <title>LayerLens Atlas SDK</title> | |
| </head> | |
| <body> | |
| <h1>LayerLens Atlas SDK</h1> | |
| <a href="${{ env.PACKAGE_NAME }}/">${{ env.PACKAGE_NAME }}</a><br/> | |
| </body> | |
| </html> | |
| EOF | |
| - name: Generate main index | |
| run: | | |
| cat > upload/index.html << EOF | |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <title>LayerLens Python Package Repository</title> | |
| <style> | |
| body { font-family: Arial, sans-serif; margin: 40px; } | |
| .install-cmd { background-color: #f5f5f5; padding: 15px; border-radius: 5px; font-family: monospace; margin: 20px 0; } | |
| .links { margin: 20px 0; } | |
| .links a { display: inline-block; margin-right: 20px; padding: 10px 15px; background-color: #007cba; color: white; text-decoration: none; border-radius: 3px; } | |
| .links a:hover { background-color: #005a87; } | |
| </style> | |
| </head> | |
| <body> | |
| <h1>LayerLens Python Package Repository</h1> | |
| <p>Custom Python package repository for ${{ env.PACKAGE_NAME }}</p> | |
| <div class="links"> | |
| <a href="package/">Browse packages</a> | |
| <a href="versions.html">View all versions</a> | |
| </div> | |
| <h2>Installation</h2> | |
| <div class="install-cmd"> | |
| # Install latest version<br/> | |
| pip install ${{ env.PACKAGE_NAME }} --index-url https://${{ secrets.CLOUDFRONT_DOMAIN }}/package/<br/><br/> | |
| # Install specific version<br/> | |
| pip install ${{ env.PACKAGE_NAME }}==1.0.0 --index-url https://${{ secrets.CLOUDFRONT_DOMAIN }}/package/ | |
| </div> | |
| <h2>Current Version</h2> | |
| <p>Latest version: <strong>${{ steps.get_version.outputs.VERSION }}</strong></p> | |
| </body> | |
| </html> | |
| EOF | |
| - name: Upload to S3 | |
| run: | | |
| # Upload everything except we don't want to delete old versions | |
| # Upload the package index (with --delete to clean up old files) | |
| aws s3 sync upload/package/ s3://${{ secrets.S3_BUCKET_NAME }}/package/ --delete | |
| # Upload version-specific files (without --delete to preserve old versions) | |
| aws s3 sync upload/versions/ s3://${{ secrets.S3_BUCKET_NAME }}/versions/ | |
| # Upload main pages (index.html, versions.html) | |
| aws s3 cp upload/index.html s3://${{ secrets.S3_BUCKET_NAME }}/index.html | |
| aws s3 cp upload/versions.html s3://${{ secrets.S3_BUCKET_NAME }}/versions.html | |
| - name: Invalidate CloudFront cache | |
| run: | | |
| aws cloudfront create-invalidation \ | |
| --distribution-id ${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }} \ | |
| --paths "/*" | |
| - name: Output installation command | |
| run: | | |
| echo "Package deployed successfully!" | |
| echo "Install with: pip install ${{ env.PACKAGE_NAME }} --index-url https://${{ secrets.CLOUDFRONT_DOMAIN }}/package/" |