From 026aed3e5ca25287c284b11bb65d1db12e2a690b Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 17 Jul 2025 12:22:18 -0700 Subject: [PATCH 1/4] ci: Update workflows and Dockerfile for pre-releases --- .github/workflows/pythonpublish.yml | 17 +++++++++++++++-- Dockerfile | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 7683cc64c0..a190e3ac44 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -3,6 +3,15 @@ name: Publish Python Packages and Official Images on: release: types: [published] + pull_request: + paths: + - '.github/workflows/pythonpublish.yml' + - 'Dockerfile' + - 'Dockerfile.connector' + - './plugins/flytekit-sqlalchemy/Dockerfile' + - './plugins/flytekit-openai/Dockerfile.batch' + - './plugins/flytekit-spark/Dockerfile' + - './plugins/flytekit-flyteinteractive/Dockerfile' jobs: deploy: @@ -39,14 +48,18 @@ jobs: VERSION=$(echo $GITHUB_REF | sed 's|refs/tags/v||') VERSION=$VERSION make -C plugins update_all_versions shell: bash - - name: Build all Plugins and publish + - name: Build all plugins + run: | + make -C plugins build_all_plugins + - name: Publish all plugins + if: ${{ github.event_name == 'release' }} env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - make -C plugins build_all_plugins make -C plugins publish_all_plugins - name: Sleep until pypi is available + if: ${{ github.event_name == 'release' }} id: pypiwait run: | # from 'refs/tags/v1.2.3 get 1.2.3' and make sure it's not an empty string diff --git a/Dockerfile b/Dockerfile index 13277d7279..38981e234d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ ARG DOCKER_IMAGE # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories. RUN apt-get update && apt-get install build-essential -y \ && pip install uv \ - && uv pip install --system --no-cache-dir -U flytekit==$VERSION \ + && uv pip install --system --no-cache-dir --pre -U flytekit==$VERSION \ kubernetes \ && apt-get clean autoclean \ && apt-get autoremove --yes \ From 932d6d814c7cc39c3ed08dc3fca9ae2ce7ba47de Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 17 Jul 2025 12:24:35 -0700 Subject: [PATCH 2/4] test Signed-off-by: Kevin Su --- .github/workflows/pythonpublish.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index a190e3ac44..1b3f446355 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -15,6 +15,7 @@ on: jobs: deploy: + if: ${{ github.event_name == 'release' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -48,18 +49,14 @@ jobs: VERSION=$(echo $GITHUB_REF | sed 's|refs/tags/v||') VERSION=$VERSION make -C plugins update_all_versions shell: bash - - name: Build all plugins - run: | - make -C plugins build_all_plugins - - name: Publish all plugins - if: ${{ github.event_name == 'release' }} + - name: Build all Plugins and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | + make -C plugins build_all_plugins make -C plugins publish_all_plugins - name: Sleep until pypi is available - if: ${{ github.event_name == 'release' }} id: pypiwait run: | # from 'refs/tags/v1.2.3 get 1.2.3' and make sure it's not an empty string From ea30be82b9f9bede730e17e0ec9de851146920cf Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 17 Jul 2025 12:40:12 -0700 Subject: [PATCH 3/4] test Signed-off-by: Kevin Su --- .github/workflows/pythonpublish.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 1b3f446355..bd8f4ffa12 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -3,15 +3,6 @@ name: Publish Python Packages and Official Images on: release: types: [published] - pull_request: - paths: - - '.github/workflows/pythonpublish.yml' - - 'Dockerfile' - - 'Dockerfile.connector' - - './plugins/flytekit-sqlalchemy/Dockerfile' - - './plugins/flytekit-openai/Dockerfile.batch' - - './plugins/flytekit-spark/Dockerfile' - - './plugins/flytekit-flyteinteractive/Dockerfile' jobs: deploy: @@ -103,7 +94,6 @@ jobs: id: buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - if: ${{ github.event_name == 'release' }} uses: docker/login-action@v3 with: registry: ghcr.io From 36aad44cc9d378991b5824d48296f702fcb89cf4 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 17 Jul 2025 12:47:35 -0700 Subject: [PATCH 4/4] pre Signed-off-by: Kevin Su --- .github/workflows/pythonpublish.yml | 2 +- Dockerfile.connector | 4 ++-- Dockerfile.dev | 2 +- plugins/flytekit-flyteinteractive/Dockerfile | 2 +- plugins/flytekit-openai/Dockerfile.batch | 2 +- plugins/flytekit-spark/Dockerfile | 2 +- plugins/flytekit-sqlalchemy/Dockerfile | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index bd8f4ffa12..7683cc64c0 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -6,7 +6,6 @@ on: jobs: deploy: - if: ${{ github.event_name == 'release' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -94,6 +93,7 @@ jobs: id: buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry + if: ${{ github.event_name == 'release' }} uses: docker/login-action@v3 with: registry: ghcr.io diff --git a/Dockerfile.connector b/Dockerfile.connector index bed0fa0160..d48c3efc0a 100644 --- a/Dockerfile.connector +++ b/Dockerfile.connector @@ -8,7 +8,7 @@ ARG VERSION RUN apt-get update && apt-get install build-essential -y \ && pip install uv -RUN uv pip install --system --no-cache-dir -U flytekit[connector]==$VERSION \ +RUN uv pip install --system --no-cache-dir -U --pre flytekit[connector]==$VERSION \ flytekitplugins-airflow==$VERSION \ flytekitplugins-bigquery==$VERSION \ flytekitplugins-k8sdataservice==$VERSION \ @@ -26,6 +26,6 @@ CMD ["pyflyte", "serve", "connector", "--port", "8000"] FROM connector-slim AS connector-all ARG VERSION -RUN uv pip install --system --no-cache-dir -U \ +RUN uv pip install --system --pre --no-cache-dir -U \ flytekitplugins-mmcloud==$VERSION \ flytekitplugins-spark==$VERSION diff --git a/Dockerfile.dev b/Dockerfile.dev index 1a839104e4..d96fa3cd67 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -35,7 +35,7 @@ COPY . /flytekit # Use a future version of SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEIDL such that uv resolution works. RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION \ SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEIDL=3.0.0dev0 \ - uv pip install --system --no-cache-dir -U \ + uv pip install --system --no-cache-dir --pre -U \ "git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl" \ -e /flytekit \ -e /flytekit/plugins/flytekit-deck-standard \ diff --git a/plugins/flytekit-flyteinteractive/Dockerfile b/plugins/flytekit-flyteinteractive/Dockerfile index e9a4105228..eba3aad035 100644 --- a/plugins/flytekit-flyteinteractive/Dockerfile +++ b/plugins/flytekit-flyteinteractive/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get update \ && wget --no-check-certificate https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server \ && wget --no-check-certificate https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server \ && pip install --no-cache-dir uv \ - && uv pip install --system --no-cache-dir -U flytekitplugins-flyteinteractive==$VERSION flytekit==$VERSION \ + && uv pip install --system --pre --no-cache-dir -U flytekitplugins-flyteinteractive==$VERSION flytekit==$VERSION \ && apt-get clean autoclean \ && apt-get autoremove --yes \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ \ diff --git a/plugins/flytekit-openai/Dockerfile.batch b/plugins/flytekit-openai/Dockerfile.batch index 39191e9f26..19e9baf8f4 100644 --- a/plugins/flytekit-openai/Dockerfile.batch +++ b/plugins/flytekit-openai/Dockerfile.batch @@ -9,7 +9,7 @@ ENV PYTHONPATH /root ARG VERSION RUN pip install uv --no-cache-dir \ - && uv pip install --system --no-cache-dir -U flytekitplugins-openai==$VERSION \ + && uv pip install --system --pre --no-cache-dir -U flytekitplugins-openai==$VERSION \ flytekit==$VERSION RUN useradd -u 1000 flytekit diff --git a/plugins/flytekit-spark/Dockerfile b/plugins/flytekit-spark/Dockerfile index 06891d2679..d86e44c6a4 100644 --- a/plugins/flytekit-spark/Dockerfile +++ b/plugins/flytekit-spark/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update && apt-get install -y wget ARG VERSION RUN pip install uv --no-cache-dir \ - && uv pip install --system --no-cache-dir -U flytekitplugins-spark==$VERSION flytekit==$VERSION + && uv pip install --system --no-cache-dir --pre -U flytekitplugins-spark==$VERSION flytekit==$VERSION RUN wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.4.0/hadoop-aws-3.4.0.jar -P /opt/spark/jars && \ wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar -P /opt/spark/jars && \ diff --git a/plugins/flytekit-sqlalchemy/Dockerfile b/plugins/flytekit-sqlalchemy/Dockerfile index c255bd6f61..99dc217e97 100644 --- a/plugins/flytekit-sqlalchemy/Dockerfile +++ b/plugins/flytekit-sqlalchemy/Dockerfile @@ -9,7 +9,7 @@ ENV PYTHONPATH /root ARG VERSION RUN pip install uv --no-cache-dir \ - && uv pip install --system --no-cache-dir -U \ + && uv pip install --system --pre --no-cache-dir -U \ sqlalchemy \ psycopg2-binary \ pymysql \