Skip to content

Commit

Permalink
Merge pull request #74 from snowplow/release/mobile/bigquery/1.0.0
Browse files Browse the repository at this point in the history
Release/mobile/bigquery/1.0.0
  • Loading branch information
bill-warner authored Apr 26, 2021
2 parents e1e10fb + ba36f9a commit ed22427
Show file tree
Hide file tree
Showing 114 changed files with 5,954 additions and 72 deletions.
2 changes: 1 addition & 1 deletion .scripts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ These scripts also require setting up a great_expectations datasource. To do so,

These scripts allow for handling passwords via two means which reduce the risk of committing credentials to source control.

1. Set the `REDSHIFT_PASSWORD` or `SNOWFLAKE_PASSWORD` environment variables for Redshift and Snowflake respectively, or the `BIGQUERY_CREDS` environment variable for Bigquery.
1. Set the `REDSHIFT_PASSWORD` or `SNOWFLAKE_PASSWORD` environment variables for Redshift and Snowflake respectively, or for BigQuery set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to path of your JSON service account key file.

2. Pass the relevant credential to the relevant argument of the script in question.

Expand Down
17 changes: 1 addition & 16 deletions .scripts/run_config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,23 +52,8 @@ if [ $target == "Default" ]; then

elif [ "$target" == "BigQuery" ]; then

BIGQUERY_CREDS=${BIGQUERY_CREDS:-$CREDENTIALS}
export GOOGLE_APPLICATION_CREDENTIALS=$root_path/tmp/bq_creds.json
export GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS:-$CREDENTIALS}

if [ -n "$BIGQUERY_CREDS" ]; then

# If creds provided via env var or argument, set trap to clean up, then create creds file.
cleanup() {
echo "run_config: Removing playbook file"
rm -f $root_path/tmp/current_playbook.yml
echo "run_config: Removing credentials file"
rm -f $root_path/tmp/bq_creds.json
}

echo "run_config: writing bq creds to file"
echo $BIGQUERY_CREDS > $root_path/tmp/bq_creds.json

fi
fi

for i in "${playbooks[@]}";
Expand Down
33 changes: 4 additions & 29 deletions .scripts/run_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,35 +22,10 @@ cd $root_path/.test

set -e

if [ "$DATABASE" == "bigquery" ]; then

BIGQUERY_CREDS=${BIGQUERY_CREDS:-$CREDENTIALS}

if [ -n "$BIGQUERY_CREDS" ]; then

# If creds provided via env var or argument, set trap to clean up, then create creds file.
cleanup() {
echo "run_test: Removing credentials file"
rm -f $root_path/tmp/bq_creds.json
}
trap cleanup EXIT

mkdir -p "${root_path}/tmp"
echo "run_test: writing bq creds to file"
echo $BIGQUERY_CREDS > $root_path/tmp/bq_creds.json

fi

# Set GOOGLE_APPLICATION_CREDENTIALS env var.
export GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS:-$root_path/tmp/bq_creds.json}

else

# If not BQ, take the relevant env var if it exists, set it to whatever's provided otherwise.
export REDSHIFT_PASSWORD=${REDSHIFT_PASSWORD:-$CREDENTIALS}
export SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD:-$CREDENTIALS}

fi
# Take the relevant env var if it exists, set it to whatever's provided otherwise.
export REDSHIFT_PASSWORD=${REDSHIFT_PASSWORD:-$CREDENTIALS}
export SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD:-$CREDENTIALS}
export GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS:-$CREDENTIALS}

# Set dummy env vars if not set already (to avoid config error)
export REDSHIFT_PASSWORD=${REDSHIFT_PASSWORD:-'dummy'}
Expand Down
145 changes: 145 additions & 0 deletions .test/great_expectations/expectations/mobile/v1/mobile_base.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
{
"data_asset_type": "Dataset",
"expectation_suite_name": "base",
"expectations": [
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "platform"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "etl_tstamp"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "collector_tstamp"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "dvce_created_tstamp"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "event"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "event_id"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "name_tracker"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "v_tracker"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "v_collector"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "v_etl"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "device_user_id"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "network_userid"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "dvce_sent_tstamp"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "session_id"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "session_index"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "derived_tstamp"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "event_vendor"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "event_name"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "event_format"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "event_version"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "event"
}
},
{
"expectation_type": "expect_column_values_to_be_unique",
"kwargs": {
"column": "event_id"
}
}
],
"meta": {
"versions": {
"test_suite_version": "1.0.1",
"bigquery_model_version": "1.0.0"
},
"great_expectations.__version__": "0.12.0"
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@
],
"meta": {
"versions": {
"test_suite_version": "1.0.0",
"test_suite_version": "1.0.1",
"redshift_model_version": "1.0.0"
},
"great_expectations.__version__": "0.12.0"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,9 @@
],
"meta": {
"versions": {
"test_suite_version": "1.0.0",
"redshift_model_version": "1.0.0"
"test_suite_version": "1.0.1",
"redshift_model_version": "1.0.0",
"bigquery_model_version": "1.0.0"
},
"great_expectations.__version__": "0.12.0"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@
],
"meta": {
"versions": {
"test_suite_version": "1.0.0",
"redshift_model_version": "1.0.0"
"test_suite_version": "1.0.1",
"redshift_model_version": "1.0.0",
"bigquery_model_version": "1.0.0"
},
"great_expectations.__version__": "0.12.0"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
{
"expectation_type": "expect_table_column_count_to_equal",
"kwargs": {
"value": 57
"value": 58
}
},
{
Expand All @@ -27,6 +27,7 @@
"dvce_created_tstamp",
"collector_tstamp",
"derived_tstamp",
"model_tstamp",
"screen_view_name",
"screen_view_transition_type",
"screen_view_type",
Expand Down Expand Up @@ -138,7 +139,12 @@
"column": "derived_tstamp"
}
},

{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
"column": "model_tstamp"
}
},
{
"expectation_type": "expect_column_values_to_not_be_null",
"kwargs": {
Expand All @@ -160,8 +166,8 @@
],
"meta": {
"versions": {
"test_suite_version": "1.0.0",
"redshift_model_version": "1.0.0"
"test_suite_version": "1.0.1",
"bigquery_model_version": "1.0.0"
},
"great_expectations.__version__": "0.12.0"
}
Expand Down
Loading

0 comments on commit ed22427

Please sign in to comment.