diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 33b2bd11..46eb8873 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,3 +35,18 @@ terraform-docs . ``` This will replace the readme file at `deploy/infrastructure/README.md` and `deploy/meltano/README.md` with any changes made to the module and header docs. + +## Developing transforms + +To develop transforms, you'll need to duplicate `.env.template` as `.env` and ensure that at least these env vars are declared: + +- `SNOWFLAKE_USER` +- `SNOWFLAKE_PASSWORD` + +## Incremental build with `--defer` option + +```console +meltano invoke dbt-snowflake:seed +meltano invoke dbt-snowflake:snapshot +meltano invoke dbt-snowflake:build +``` diff --git a/data/.env.template b/data/.env.template index 50819b03..342d6683 100644 --- a/data/.env.template +++ b/data/.env.template @@ -19,10 +19,9 @@ PERMISSION_BOT_PASSWORD="****" PERMISSION_BOT_ACCOUNT="****" # Snowflake MELTANO User Password +SNOWFLAKE_USER="...." SNOWFLAKE_PASSWORD="****" - -# dbt Snowflake Password -DBT_SNOWFLAKE_PASSWORD="****" +SNOWFLAKE_ROLE="DEVELOPER" # Slack Webhooks TARGET_APPRISE_SINGER_ACTIVITY_URIS=["https://hooks.slack.com/services/{}/{}/{}"] diff --git a/data/.sqlfluff b/data/.sqlfluff index e764e243..d9cf2193 100644 --- a/data/.sqlfluff +++ b/data/.sqlfluff @@ -17,7 +17,9 @@ profile = meltano tab_space_size = 4 max_line_length = 80 indent_unit = space -comma_style = trailing + +[sqlfluff:layout:type:comma] +line_position = trailing [sqlfluff:rules:L010] # Keywords capitalisation_policy = upper diff --git a/data/environments/dbt_dev.meltano.yml b/data/environments/dbt_dev.meltano.yml new file mode 100644 index 00000000..00fa72c9 --- /dev/null +++ b/data/environments/dbt_dev.meltano.yml @@ -0,0 +1,42 @@ +environments: +- name: dbt_dev + annotations: + docs: + description: > + Environment for developing dbt transforms separately from EL. + config: + plugins: + utilities: + - name: dbt-snowflake + config: + user: ${SNOWFLAKE_USER} + role: ${SNOWFLAKE_USER} + warehouse: CORE + skip_pre_invoke: true + database: USERDEV_PROD + database_prep: USERDEV_PREP + target_schema_prefix: ${SNOWFLAKE_USER}_ + - name: sqlfluff + config: + user: ${SNOWFLAKE_USER} + - name: great_expectations + config: + prod_database: USERDEV_PROD + raw_database: USERDEV_RAW + username: ${SNOWFLAKE_USER} + role: ${SNOWFLAKE_USER} + warehouse: CORE + env: + USER_PREFIX: ${SNOWFLAKE_USER} + SUPERSET_API_URL: http://localhost:8088 + SUPERSET_USER: admin + SUPERSET_PASS: admin + # https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html + AIRFLOW__CORE__PLUGINS_FOLDER: $MELTANO_PROJECT_ROOT/orchestrate/plugins_local + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: '30' + AIRFLOW_VAR_MELTANO_ENVIRONMENT: userdev + AIRFLOW_VAR_OPERATOR_TYPE: bash + # Secrets via KMS + KMS_PUBLIC_KEY_PATH: utilities/kms/Publickey.pem + KMS_DOTENV_PATH: .env + KMS_SECRETS_PATH: secrets.yml diff --git a/data/environments/userdev.meltano.yml b/data/environments/userdev.meltano.yml index 79e8f760..bbd8ced4 100644 --- a/data/environments/userdev.meltano.yml +++ b/data/environments/userdev.meltano.yml @@ -22,8 +22,8 @@ environments: - name: tap-snowflake config: dbname: USERDEV_PROD - user: ${USER_PREFIX} - role: ${USER_PREFIX} + user: ${SNOWFLAKE_USER} + role: ${SNOWFLAKE_USER} warehouse: CORE - name: tap-snowflake-metrics-legacy config: @@ -85,7 +85,7 @@ environments: role: ${USER_PREFIX} warehouse: CORE env: - USER_PREFIX: PNADOLNY + USER_PREFIX: ${SNOWFLAKE_USER} SUPERSET_API_URL: http://localhost:8088 SUPERSET_USER: admin SUPERSET_PASS: admin @@ -97,4 +97,4 @@ environments: # Secrets via KMS KMS_PUBLIC_KEY_PATH: utilities/kms/Publickey.pem KMS_DOTENV_PATH: .env - KMS_SECTRETS_PATH: secrets.yml + KMS_SECRETS_PATH: secrets.yml diff --git a/data/meltano.yml b/data/meltano.yml index 6122bd80..9ad0fb0f 100644 --- a/data/meltano.yml +++ b/data/meltano.yml @@ -1,5 +1,5 @@ version: 1 -default_environment: userdev +default_environment: dbt_dev send_anonymous_usage_stats: false project_id: c15e971a-d318-4a9d-979b-1039ce5fd1b1 include_paths: @@ -9,3 +9,5 @@ include_paths: - ./orchestrate/*.meltano.yml - ./transform/*.meltano.yml - ./utilities/*.meltano.yml +ff: + strict_env_var_mode: true diff --git a/data/transform/profiles/snowflake/profiles.yml b/data/transform/profiles/snowflake/profiles.yml index df7b0fbb..23b47043 100644 --- a/data/transform/profiles/snowflake/profiles.yml +++ b/data/transform/profiles/snowflake/profiles.yml @@ -18,6 +18,16 @@ meltano: warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" schema: "DEFAULT" + dbt_dev: + type: snowflake + threads: "{{ env_var('DBT_THREADS', 6) | as_number }}" + account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_USER') }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + role: "{{ env_var('DBT_SNOWFLAKE_ROLE', env_var('SNOWFLAKE_USER', )) }}" + warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" + database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" + schema: "DEFAULT" userdev: type: snowflake threads: 6 diff --git a/data/transform/transformers.meltano.yml b/data/transform/transformers.meltano.yml index 21e91564..792062cf 100644 --- a/data/transform/transformers.meltano.yml +++ b/data/transform/transformers.meltano.yml @@ -38,6 +38,7 @@ plugins: run_slack_notifications: run --select +publish.slack_notifications.* run_hubspot_publish: run --select publish.hubspot.* test_hubspot_publish: test --select publish.hubspot.* + build: run --fail-fast --defer --state=${MELTANO_PROJECT_ROOT}/.meltano/transformers/dbt/target/ config: account: epa06486 diff --git a/data/utilities/utilities.meltano.yml b/data/utilities/utilities.meltano.yml index 50ed631f..b1460469 100644 --- a/data/utilities/utilities.meltano.yml +++ b/data/utilities/utilities.meltano.yml @@ -68,9 +68,9 @@ plugins: executable: kms commands: encrypt: encrypt $KMS_PUBLIC_KEY_PATH --dotenv-path $KMS_DOTENV_PATH --output-path - $KMS_SECTRETS_PATH + $KMS_SECRETS_PATH decrypt: decrypt $KMS_KEY_ID --input-path $KMS_SECRETS_PATH --output-path $KMS_DOTENV_PATH env: KMS_PUBLIC_KEY_PATH: utilities/kms/Publickey.pem KMS_DOTENV_PATH: .env - KMS_SECTRETS_PATH: secrets.yml + KMS_SECRETS_PATH: secrets.yml