diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b95ea58f..cf6ae198 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,46 @@ on: workflow_dispatch: jobs: + # Jobs to run change detection + changes: + runs-on: ubuntu-latest + # Set job outputs to values from filter step + outputs: + orchestrate: ${{ steps.filter.outputs.orchestrate }} + el: ${{ steps.filter.outputs.el }} + sql_lint: ${{ steps.filter.outputs.sql_lint }} + sql: ${{ steps.filter.outputs.sql }} + generic: ${{ steps.filter.outputs.generic }} + + steps: + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + orchestrate: + - 'data/orchestrate/**.yml' + - 'data/orchestrate/**.py' + el: + - '/extract/*.yml' + - '/load/*.yml' + - 'data/transform/**.sql' + - 'data/transform/**.yml' + sql_lint: + - 'data/.sqlfluff' + - 'data/.sqlfluffignore' + - 'data/transform/**.sql' + - 'data/utilities/utilities.meltano.yml' + sql: + - 'data/transform/**.sql' + - 'data/transform/**.yml' + generic: + - 'data/plugins/**' + - 'data/environment/*.yml' + airflow_tests: + needs: changes + if: ${{ needs.changes.outputs.orchestrate == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }} runs-on: ubuntu-latest defaults: run: @@ -31,6 +70,8 @@ jobs: - run: meltano test airflow sqlfluff_tests: + needs: changes + if: ${{ needs.changes.outputs.sql_lint == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }} runs-on: ubuntu-latest defaults: run: @@ -56,6 +97,8 @@ jobs: - run: meltano run sqlfluff:lint dbt_seed: + needs: changes + if: ${{ needs.changes.outputs.el == 'true' }} || ${{ needs.changes.outputs.sql == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }} runs-on: ubuntu-latest defaults: run: @@ -77,10 +120,12 @@ jobs: # Run Test - run: meltano install transformer dbt-snowflake - run: meltano run dbt-snowflake:deps dbt-snowflake:seed + - run: meltano run dbt-snowflake:clone_snapshots el_tests: + needs: [airflow_tests, sqlfluff_tests, dbt_seed, changes] + if: ${{ needs.changes.outputs.el == 'true' }} || ${{ needs.changes.outputs.sql == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }} runs-on: ubuntu-latest - needs: [airflow_tests, sqlfluff_tests, dbt_seed] defaults: run: working-directory: data/ @@ -107,8 +152,9 @@ jobs: - run: meltano run dbt-snowflake:deps ${{ matrix.extract-source }} transform_tests: + needs: [el_tests, changes] + if: ${{ needs.changes.outputs.el == 'true' }} || ${{ needs.changes.outputs.sql == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }} runs-on: ubuntu-latest - needs: [el_tests] defaults: run: working-directory: data/ @@ -131,8 +177,9 @@ jobs: - run: meltano run dbt-snowflake:deps dbt-snowflake:run dbt-snowflake:test reverse_el_tests: + needs: [transform_tests, changes] + if: ${{ needs.changes.outputs.el == 'true' }} || ${{ needs.changes.outputs.sql == 'true' }} || ${{ needs.changes.outputs.generic == 'true' }} runs-on: ubuntu-latest - needs: [transform_tests] defaults: run: working-directory: data/