From 7518f4a1706109e7a9dcbae1e823f731ea0ccb1a Mon Sep 17 00:00:00 2001 From: Peeter Piegaze <1153481+ppiegaze@users.noreply.github.com> Date: Mon, 20 Jan 2025 15:39:42 +0100 Subject: [PATCH 1/5] update Signed-off-by: Peeter Piegaze <1153481+ppiegaze@users.noreply.github.com> --- .../workflows/workflows.py | 98 ------------------- .../cookiecutter.json | 0 .../.python-version | 0 .../{{cookiecutter.project_name}}/LICENSE | 0 .../{{cookiecutter.project_name}}/README.md | 0 .../pyproject.toml | 0 .../{{cookiecutter.project_name}}/uv.lock | 0 .../workflows/__init__.py | 0 .../workflows/hello-world.py | 33 +++++++ .../cookiecutter.json | 0 .../{{cookiecutter.project_name}}/.gitignore | 0 .../.python-version | 0 .../{{cookiecutter.project_name}}/LICENSE | 0 .../{{cookiecutter.project_name}}/README.md | 0 .../docs/docs.md | 0 .../pyproject.toml | 0 .../src/core/__init__.py | 0 .../src/core/core.py | 0 .../src/orchestration/__init__.py | 0 .../src/orchestration/orchestration.py | 0 .../src/tasks/__init__.py | 0 .../src/tasks/tasks.py | 2 +- .../src/workflows/__init__.py | 0 .../src/workflows/workflows.py | 0 .../{{cookiecutter.project_name}}/uv.lock | 0 25 files changed, 34 insertions(+), 99 deletions(-) delete mode 100644 basic-union-template/{{cookiecutter.project_name}}/workflows/workflows.py rename {basic-union-template => union-hello-world}/cookiecutter.json (100%) rename {basic-union-template => union-hello-world}/{{cookiecutter.project_name}}/.python-version (100%) rename {basic-union-template => union-hello-world}/{{cookiecutter.project_name}}/LICENSE (100%) rename {basic-union-template => union-hello-world}/{{cookiecutter.project_name}}/README.md (100%) rename {basic-union-template => union-hello-world}/{{cookiecutter.project_name}}/pyproject.toml (100%) rename {basic-union-template => union-hello-world}/{{cookiecutter.project_name}}/uv.lock (100%) rename {basic-union-template => union-hello-world}/{{cookiecutter.project_name}}/workflows/__init__.py (100%) create mode 100644 union-hello-world/{{cookiecutter.project_name}}/workflows/hello-world.py rename {first-project => union-standard}/cookiecutter.json (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/.gitignore (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/.python-version (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/LICENSE (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/README.md (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/docs/docs.md (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/pyproject.toml (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/src/core/__init__.py (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/src/core/core.py (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/src/orchestration/__init__.py (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/src/orchestration/orchestration.py (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/src/tasks/__init__.py (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/src/tasks/tasks.py (96%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/src/workflows/__init__.py (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/src/workflows/workflows.py (100%) rename {first-project => union-standard}/{{cookiecutter.project_name}}/uv.lock (100%) diff --git a/basic-union-template/{{cookiecutter.project_name}}/workflows/workflows.py b/basic-union-template/{{cookiecutter.project_name}}/workflows/workflows.py deleted file mode 100644 index 17c18c6..0000000 --- a/basic-union-template/{{cookiecutter.project_name}}/workflows/workflows.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Workflows""" - -import union - -# ImageSpec defines the container image used for the Kubernetes pods that run the tasks in Union. -image_spec = union.ImageSpec( - - # The name of the image - name="basic-union-image", - - # Use the requirements.txt to define the packages to be installed in the the image - requirements="uv.lock", - - # Container registry to which the image will be pushed. - # - # ON UNION BYOC UNCOMMENT THIS PARAMETER! - # - # Make sure that: - # - # * You substitute the actual name of the registry here. - # (for example if you are using GitHub's GHCR, you would - # use "ghcr.io/"). - # - # * You have Docker installed locally and are logged into the registry. - # - # * The image, once pushed to the registry, is accessible to Union - # (for example, for GHCR, make sure the image is public) - # - # This parameter is only needed for BYOC. On Serverless, images are stored - # transparently in Union's own container registry. - # registry="" - - # The base image on which this image is based - # base_image="/" - - # Python version of the image. Use default python in the base image if None. - # python_version="3.11" - - # Plugin used to build the image locally in Union BYOC. - # Uses flytekitplugin_envd by default. - # - # This only applies to BYOC. On Serverless, images are built by Union's ImageBuilder service. - # builder="flytekitplugins_envd" - - # Source root of the image. - # source_root="/path/to/source/root" - - # Environment variables to be set in the image. - # env=[FOO="foo", BAR="bar"] - - # List of python packages to install. - # packages=["package-1", "package-2"] - - # List of conda packages to install. - # conda_packages=["package-1", "package-2"] - - # List of conda channels. - # conda_channels=["channel-1", "channel-2"] - - # List of apt packages to install. - # apt_packages=["package-1", "package-2"] - - # Version of cuda to install. - # cuda="12.5" - - # Version of cudnn to install. - # cudnn="9.2.0" - - # Target platforms for the build output - # (for example: windows/amd64, linux/amd64, darwin/arm64) - # platform=["linux/amd64"] - - # Custom pip index url - # pip_index="https://my-pypi.mywebsite.com/simple" - - # One or more pip index urls as a list - # pip_extra_index_url=["https://my-pypi.mywebsite.com/simple", "https://my-other-pypi.mywebsite.com/simple"] - - # Path to a JSON registry config file - # registry_config ="/path/to/registry/config.json" - - # Commands to run during the build process - # commands= ["pip install -r requirements.txt"] - - # Custom string format for image tag. The ImageSpec hash is passed in as `spec_hash`. - # So, for example, to add a "dev" suffix to the image tag use: - # tag_format = "{spec_hash}-dev" -) - -@union.task(container_image=image_spec) -def say_hello(name: str) -> str: - return f"Hello, {name}!" - - -@union.workflow -def wf(name: str = "world") -> str: - greeting = say_hello(name=name) - return greeting diff --git a/basic-union-template/cookiecutter.json b/union-hello-world/cookiecutter.json similarity index 100% rename from basic-union-template/cookiecutter.json rename to union-hello-world/cookiecutter.json diff --git a/basic-union-template/{{cookiecutter.project_name}}/.python-version b/union-hello-world/{{cookiecutter.project_name}}/.python-version similarity index 100% rename from basic-union-template/{{cookiecutter.project_name}}/.python-version rename to union-hello-world/{{cookiecutter.project_name}}/.python-version diff --git a/basic-union-template/{{cookiecutter.project_name}}/LICENSE b/union-hello-world/{{cookiecutter.project_name}}/LICENSE similarity index 100% rename from basic-union-template/{{cookiecutter.project_name}}/LICENSE rename to union-hello-world/{{cookiecutter.project_name}}/LICENSE diff --git a/basic-union-template/{{cookiecutter.project_name}}/README.md b/union-hello-world/{{cookiecutter.project_name}}/README.md similarity index 100% rename from basic-union-template/{{cookiecutter.project_name}}/README.md rename to union-hello-world/{{cookiecutter.project_name}}/README.md diff --git a/basic-union-template/{{cookiecutter.project_name}}/pyproject.toml b/union-hello-world/{{cookiecutter.project_name}}/pyproject.toml similarity index 100% rename from basic-union-template/{{cookiecutter.project_name}}/pyproject.toml rename to union-hello-world/{{cookiecutter.project_name}}/pyproject.toml diff --git a/basic-union-template/{{cookiecutter.project_name}}/uv.lock b/union-hello-world/{{cookiecutter.project_name}}/uv.lock similarity index 100% rename from basic-union-template/{{cookiecutter.project_name}}/uv.lock rename to union-hello-world/{{cookiecutter.project_name}}/uv.lock diff --git a/basic-union-template/{{cookiecutter.project_name}}/workflows/__init__.py b/union-hello-world/{{cookiecutter.project_name}}/workflows/__init__.py similarity index 100% rename from basic-union-template/{{cookiecutter.project_name}}/workflows/__init__.py rename to union-hello-world/{{cookiecutter.project_name}}/workflows/__init__.py diff --git a/union-hello-world/{{cookiecutter.project_name}}/workflows/hello-world.py b/union-hello-world/{{cookiecutter.project_name}}/workflows/hello-world.py new file mode 100644 index 0000000..2dfa42a --- /dev/null +++ b/union-hello-world/{{cookiecutter.project_name}}/workflows/hello-world.py @@ -0,0 +1,33 @@ +import union + +image_spec = union.ImageSpec( + + # The name of the image. + name="hello-world-image", + + # Use the `uv.lock` file in this project to define the dependencies included in the image. + requirements="uv.lock", + + # The container registry to which the image will be pushed. + # Only used for Union BYOC. Not used for Union Serverless + # Uncomment this parameter if you are using Union BYOC.: + # + # * Substitute the actual name of the registry for . + # (for example if you are using GitHub's GHCR, you would use "ghcr.io/"). + # + # * Make sure you have Docker installed locally and are logged into that registry. + # + # * Make sure that the image, once pushed to the registry, is accessible to Union + # (for example, for GHCR, make sure the image is public). + # + # registry="" +) + +@union.task(container_image=image_spec) +def hello_world_task(name: str) -> str: + return f"Hello, {name}!" + +@union.workflow +def hello_world_wf(name: str = "world") -> str: + greeting = say_hello(name=name) + return greeting \ No newline at end of file diff --git a/first-project/cookiecutter.json b/union-standard/cookiecutter.json similarity index 100% rename from first-project/cookiecutter.json rename to union-standard/cookiecutter.json diff --git a/first-project/{{cookiecutter.project_name}}/.gitignore b/union-standard/{{cookiecutter.project_name}}/.gitignore similarity index 100% rename from first-project/{{cookiecutter.project_name}}/.gitignore rename to union-standard/{{cookiecutter.project_name}}/.gitignore diff --git a/first-project/{{cookiecutter.project_name}}/.python-version b/union-standard/{{cookiecutter.project_name}}/.python-version similarity index 100% rename from first-project/{{cookiecutter.project_name}}/.python-version rename to union-standard/{{cookiecutter.project_name}}/.python-version diff --git a/first-project/{{cookiecutter.project_name}}/LICENSE b/union-standard/{{cookiecutter.project_name}}/LICENSE similarity index 100% rename from first-project/{{cookiecutter.project_name}}/LICENSE rename to union-standard/{{cookiecutter.project_name}}/LICENSE diff --git a/first-project/{{cookiecutter.project_name}}/README.md b/union-standard/{{cookiecutter.project_name}}/README.md similarity index 100% rename from first-project/{{cookiecutter.project_name}}/README.md rename to union-standard/{{cookiecutter.project_name}}/README.md diff --git a/first-project/{{cookiecutter.project_name}}/docs/docs.md b/union-standard/{{cookiecutter.project_name}}/docs/docs.md similarity index 100% rename from first-project/{{cookiecutter.project_name}}/docs/docs.md rename to union-standard/{{cookiecutter.project_name}}/docs/docs.md diff --git a/first-project/{{cookiecutter.project_name}}/pyproject.toml b/union-standard/{{cookiecutter.project_name}}/pyproject.toml similarity index 100% rename from first-project/{{cookiecutter.project_name}}/pyproject.toml rename to union-standard/{{cookiecutter.project_name}}/pyproject.toml diff --git a/first-project/{{cookiecutter.project_name}}/src/core/__init__.py b/union-standard/{{cookiecutter.project_name}}/src/core/__init__.py similarity index 100% rename from first-project/{{cookiecutter.project_name}}/src/core/__init__.py rename to union-standard/{{cookiecutter.project_name}}/src/core/__init__.py diff --git a/first-project/{{cookiecutter.project_name}}/src/core/core.py b/union-standard/{{cookiecutter.project_name}}/src/core/core.py similarity index 100% rename from first-project/{{cookiecutter.project_name}}/src/core/core.py rename to union-standard/{{cookiecutter.project_name}}/src/core/core.py diff --git a/first-project/{{cookiecutter.project_name}}/src/orchestration/__init__.py b/union-standard/{{cookiecutter.project_name}}/src/orchestration/__init__.py similarity index 100% rename from first-project/{{cookiecutter.project_name}}/src/orchestration/__init__.py rename to union-standard/{{cookiecutter.project_name}}/src/orchestration/__init__.py diff --git a/first-project/{{cookiecutter.project_name}}/src/orchestration/orchestration.py b/union-standard/{{cookiecutter.project_name}}/src/orchestration/orchestration.py similarity index 100% rename from first-project/{{cookiecutter.project_name}}/src/orchestration/orchestration.py rename to union-standard/{{cookiecutter.project_name}}/src/orchestration/orchestration.py diff --git a/first-project/{{cookiecutter.project_name}}/src/tasks/__init__.py b/union-standard/{{cookiecutter.project_name}}/src/tasks/__init__.py similarity index 100% rename from first-project/{{cookiecutter.project_name}}/src/tasks/__init__.py rename to union-standard/{{cookiecutter.project_name}}/src/tasks/__init__.py diff --git a/first-project/{{cookiecutter.project_name}}/src/tasks/tasks.py b/union-standard/{{cookiecutter.project_name}}/src/tasks/tasks.py similarity index 96% rename from first-project/{{cookiecutter.project_name}}/src/tasks/tasks.py rename to union-standard/{{cookiecutter.project_name}}/src/tasks/tasks.py index 95a8a01..4c486f5 100644 --- a/first-project/{{cookiecutter.project_name}}/src/tasks/tasks.py +++ b/union-standard/{{cookiecutter.project_name}}/src/tasks/tasks.py @@ -5,7 +5,7 @@ image_spec = union.ImageSpec( # The name of the image. - name="first-project-image", + name="standard-image", # Use the `uv.lock` file in this project to define the dependencies included in the image. requirements="uv.lock", diff --git a/first-project/{{cookiecutter.project_name}}/src/workflows/__init__.py b/union-standard/{{cookiecutter.project_name}}/src/workflows/__init__.py similarity index 100% rename from first-project/{{cookiecutter.project_name}}/src/workflows/__init__.py rename to union-standard/{{cookiecutter.project_name}}/src/workflows/__init__.py diff --git a/first-project/{{cookiecutter.project_name}}/src/workflows/workflows.py b/union-standard/{{cookiecutter.project_name}}/src/workflows/workflows.py similarity index 100% rename from first-project/{{cookiecutter.project_name}}/src/workflows/workflows.py rename to union-standard/{{cookiecutter.project_name}}/src/workflows/workflows.py diff --git a/first-project/{{cookiecutter.project_name}}/uv.lock b/union-standard/{{cookiecutter.project_name}}/uv.lock similarity index 100% rename from first-project/{{cookiecutter.project_name}}/uv.lock rename to union-standard/{{cookiecutter.project_name}}/uv.lock From c7f1182ff570578fcb29116cccb1d059563e529b Mon Sep 17 00:00:00 2001 From: Peeter Piegaze <1153481+ppiegaze@users.noreply.github.com> Date: Mon, 20 Jan 2025 16:12:53 +0100 Subject: [PATCH 2/5] try to fix integration test failure Signed-off-by: Peeter Piegaze <1153481+ppiegaze@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8550e0d..13f2fec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ flytekit argparse -torch[cpu]>=2.0,<2.1 +torch[cpu]>=2.0,<2.3 torchvision>=0.15,<0.16 pandas==1.5.3 scikit-learn==1.2.2 From da39821c3921447869103dc396f0581b8d1d3ba6 Mon Sep 17 00:00:00 2001 From: Peeter Piegaze <1153481+ppiegaze@users.noreply.github.com> Date: Mon, 20 Jan 2025 16:21:36 +0100 Subject: [PATCH 3/5] try again Signed-off-by: Peeter Piegaze <1153481+ppiegaze@users.noreply.github.com> --- .github/workflows/deployImage.yml | 2 +- .github/workflows/runIntegration.yml | 4 ++-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deployImage.yml b/.github/workflows/deployImage.yml index b9bf7e3..e5f5efb 100644 --- a/.github/workflows/deployImage.yml +++ b/.github/workflows/deployImage.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Log in to GitHub Container Registry uses: docker/login-action@v1 diff --git a/.github/workflows/runIntegration.yml b/.github/workflows/runIntegration.yml index 9ae21be..e6738fb 100644 --- a/.github/workflows/runIntegration.yml +++ b/.github/workflows/runIntegration.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 tags: true @@ -73,7 +73,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Run integration tests env: host: ${{ secrets.FLYTE_HOST }} diff --git a/requirements.txt b/requirements.txt index 13f2fec..8550e0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ flytekit argparse -torch[cpu]>=2.0,<2.3 +torch[cpu]>=2.0,<2.1 torchvision>=0.15,<0.16 pandas==1.5.3 scikit-learn==1.2.2 From 6af6a74fbe7f29d55b5ed7ce9649b44cc7add2bf Mon Sep 17 00:00:00 2001 From: Peeter Piegaze <1153481+ppiegaze@users.noreply.github.com> Date: Mon, 20 Jan 2025 17:32:20 +0100 Subject: [PATCH 4/5] set python to v=3.12 in github actions Signed-off-by: Peeter Piegaze <1153481+ppiegaze@users.noreply.github.com> --- .github/workflows/runIntegration.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/runIntegration.yml b/.github/workflows/runIntegration.yml index e6738fb..431570b 100644 --- a/.github/workflows/runIntegration.yml +++ b/.github/workflows/runIntegration.yml @@ -15,14 +15,15 @@ jobs: fetch-depth: 0 tags: true token: ${{ secrets.FLYTE_BOT_PAT }} - + - uses: actions/setup-python@v5 + with: + python-version: '3.12' - name: Log in to GitHub Container Registry uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ secrets.FLYTE_BOT_USERNAME }} password: ${{ secrets.FLYTE_BOT_PAT }} - - name: Iterate through top level directories and build Docker images env: ACCESS_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} @@ -74,6 +75,9 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' - name: Run integration tests env: host: ${{ secrets.FLYTE_HOST }} @@ -86,4 +90,4 @@ jobs: --client_id $client_id \ --client_secret $client_secret \ --image_suffix pr-${{ github.event.pull_request.number }} \ - --image_hostname ghcr.io/${{ github.repository }} \ No newline at end of file + --image_hostname ghcr.io/${{ github.repository }} From 26ee12a4e6e1c4c16b9938e493e8f594d27ca74b Mon Sep 17 00:00:00 2001 From: Peeter Piegaze <1153481+ppiegaze@users.noreply.github.com> Date: Mon, 20 Jan 2025 17:34:25 +0100 Subject: [PATCH 5/5] try python 3.11 Signed-off-by: Peeter Piegaze <1153481+ppiegaze@users.noreply.github.com> --- .github/workflows/runIntegration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runIntegration.yml b/.github/workflows/runIntegration.yml index 431570b..4fec78a 100644 --- a/.github/workflows/runIntegration.yml +++ b/.github/workflows/runIntegration.yml @@ -17,7 +17,7 @@ jobs: token: ${{ secrets.FLYTE_BOT_PAT }} - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.11' - name: Log in to GitHub Container Registry uses: docker/login-action@v1 with: @@ -77,7 +77,7 @@ jobs: uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.11' - name: Run integration tests env: host: ${{ secrets.FLYTE_HOST }}