Skip to content

add ci check based on file path changes #357

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
pnadolny13 wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

add ci check based on file path changes #357

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 50 additions & 3 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,46 @@ on:
workflow_dispatch:

jobs:
# Jobs to run change detection
changes:
runs-on: ubuntu-latest
# Set job outputs to values from filter step
outputs:
orchestrate: ${{ steps.filter.outputs.orchestrate }}
el: ${{ steps.filter.outputs.el }}
sql_lint: ${{ steps.filter.outputs.sql_lint }}
sql: ${{ steps.filter.outputs.sql }}
generic: ${{ steps.filter.outputs.generic }}

steps:
# For pull requests it's not necessary to checkout the code
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
orchestrate:
- 'data/orchestrate/**.yml'
- 'data/orchestrate/**.py'
el:
- '/extract/*.yml'
- '/load/*.yml'
- 'data/transform/**.sql'
- 'data/transform/**.yml'
sql_lint:
- 'data/.sqlfluff'
- 'data/.sqlfluffignore'
- 'data/transform/**.sql'
- 'data/utilities/utilities.meltano.yml'
sql:
- 'data/transform/**.sql'
- 'data/transform/**.yml'
generic:
- 'data/plugins/**'
- 'data/environment/*.yml'

airflow_tests:
needs: changes
if: ${{ needs.changes.outputs.orchestrate == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }}
runs-on: ubuntu-latest
defaults:
run:
Expand All @@ -31,6 +70,8 @@ jobs:
- run: meltano test airflow

sqlfluff_tests:
needs: changes
if: ${{ needs.changes.outputs.sql_lint == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }}
runs-on: ubuntu-latest
defaults:
run:
Expand All @@ -56,6 +97,8 @@ jobs:
- run: meltano run sqlfluff:lint

dbt_seed:
needs: changes
if: ${{ needs.changes.outputs.el == 'true' }} || ${{ needs.changes.outputs.sql == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }}
runs-on: ubuntu-latest
defaults:
run:
Expand All @@ -77,10 +120,12 @@ jobs:
# Run Test
- run: meltano install transformer dbt-snowflake
- run: meltano run dbt-snowflake:deps dbt-snowflake:seed
- run: meltano run dbt-snowflake:clone_snapshots

el_tests:
needs: [airflow_tests, sqlfluff_tests, dbt_seed, changes]
if: ${{ needs.changes.outputs.el == 'true' }} || ${{ needs.changes.outputs.sql == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }}
runs-on: ubuntu-latest
needs: [airflow_tests, sqlfluff_tests, dbt_seed]
defaults:
run:
working-directory: data/
Expand All @@ -107,8 +152,9 @@ jobs:
- run: meltano run dbt-snowflake:deps ${{ matrix.extract-source }}

transform_tests:
needs: [el_tests, changes]
if: ${{ needs.changes.outputs.el == 'true' }} || ${{ needs.changes.outputs.sql == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }}
runs-on: ubuntu-latest
needs: [el_tests]
defaults:
run:
working-directory: data/
Expand All @@ -131,8 +177,9 @@ jobs:
- run: meltano run dbt-snowflake:deps dbt-snowflake:run dbt-snowflake:test

reverse_el_tests:
needs: [transform_tests, changes]
if: ${{ needs.changes.outputs.el == 'true' }} || ${{ needs.changes.outputs.sql == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }}
runs-on: ubuntu-latest
needs: [transform_tests]
defaults:
run:
working-directory: data/
Expand Down