diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_base.json b/.test/great_expectations/expectations/mobile/v1/mobile_base.json index aa13f11b..bb2a38db 100644 --- a/.test/great_expectations/expectations/mobile/v1/mobile_base.json +++ b/.test/great_expectations/expectations/mobile/v1/mobile_base.json @@ -137,7 +137,7 @@ ], "meta": { "versions": { - "test_suite_version": "1.0.1", + "test_suite_version": "1.0.2", "bigquery_model_version": "1.0.0", "snowflake_model_version": "1.0.0" }, diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_base_redshift.json b/.test/great_expectations/expectations/mobile/v1/mobile_base_redshift.json index 1c6054a7..b84a9104 100644 --- a/.test/great_expectations/expectations/mobile/v1/mobile_base_redshift.json +++ b/.test/great_expectations/expectations/mobile/v1/mobile_base_redshift.json @@ -248,8 +248,8 @@ ], "meta": { "versions": { - "test_suite_version": "1.0.1", - "redshift_model_version": "1.0.0" + "test_suite_version": "1.0.2", + "redshift_model_version": "1.1.0" }, "great_expectations.__version__": "0.12.0" } diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_metadata.json b/.test/great_expectations/expectations/mobile/v1/mobile_metadata.json index 83102feb..aeb90ebb 100644 --- a/.test/great_expectations/expectations/mobile/v1/mobile_metadata.json +++ b/.test/great_expectations/expectations/mobile/v1/mobile_metadata.json @@ -102,8 +102,8 @@ ], "meta": { "versions": { - "test_suite_version": "1.0.1", - "redshift_model_version": "1.0.0", + "test_suite_version": "1.0.2", + "redshift_model_version": "1.1.0", "bigquery_model_version": "1.0.0", "snowflake_model_version": "1.0.0" }, diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_screen_view_in_session_values.json b/.test/great_expectations/expectations/mobile/v1/mobile_screen_view_in_session_values.json index 2475ab3c..6b5a5f32 100644 --- a/.test/great_expectations/expectations/mobile/v1/mobile_screen_view_in_session_values.json +++ b/.test/great_expectations/expectations/mobile/v1/mobile_screen_view_in_session_values.json @@ -26,8 +26,8 @@ ], "meta": { "versions": { - "test_suite_version": "1.0.1", - "redshift_model_version": "1.0.0", + "test_suite_version": "1.0.2", + "redshift_model_version": "1.1.0", "bigquery_model_version": "1.0.0", "snowflake_model_version": "1.0.0" }, diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_screen_views.json b/.test/great_expectations/expectations/mobile/v1/mobile_screen_views.json index 55fbb350..84428975 100644 --- a/.test/great_expectations/expectations/mobile/v1/mobile_screen_views.json +++ b/.test/great_expectations/expectations/mobile/v1/mobile_screen_views.json @@ -166,7 +166,8 @@ ], "meta": { "versions": { - "test_suite_version": "1.0.1", + "test_suite_version": "1.0.2", + "redshift_model_version": "1.1.0", "bigquery_model_version": "1.0.0", "snowflake_model_version": "1.0.0" }, diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_screen_views_redshift.json b/.test/great_expectations/expectations/mobile/v1/mobile_screen_views_redshift.json deleted file mode 100644 index 2ac5f6b0..00000000 --- a/.test/great_expectations/expectations/mobile/v1/mobile_screen_views_redshift.json +++ /dev/null @@ -1,168 +0,0 @@ -{ - "data_asset_type": "Dataset", - "expectation_suite_name": "mobile_screen_views", - "expectations": [ - { - "expectation_type": "expect_table_column_count_to_equal", - "kwargs": { - "value": 57 - } - }, - { - "expectation_type": "expect_table_columns_to_match_ordered_list", - "kwargs": { - "column_list": [ - "screen_view_id", - "event_id", - "app_id", - "user_id", - "device_user_id", - "network_userid", - "session_id", - "session_index", - "previous_session_id", - "session_first_event_id", - "screen_view_in_session_index", - "screen_views_in_session", - "dvce_created_tstamp", - "collector_tstamp", - "derived_tstamp", - "screen_view_name", - "screen_view_transition_type", - "screen_view_type", - "screen_fragment", - "screen_top_view_controller", - "screen_view_controller", - "screen_view_previous_id", - "screen_view_previous_name", - "screen_view_previous_type", - "platform", - "dvce_screenwidth", - "dvce_screenheight", - "device_manufacturer", - "device_model", - "os_type", - "os_version", - "android_idfa", - "apple_idfa", - "apple_idfv", - "open_idfa", - "device_latitude", - "device_longitude", - "device_latitude_longitude_accuracy", - "device_altitude", - "device_altitude_accuracy", - "device_bearing", - "device_speed", - "geo_country", - "geo_region", - "geo_city", - "geo_zipcode", - "geo_latitude", - "geo_longitude", - "geo_region_name", - "geo_timezone", - "user_ipaddress", - "useragent", - "carrier", - "network_technology", - "network_type", - "build", - "version" - ] - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "screen_view_id" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "event_id" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "device_user_id" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "network_userid" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "session_id" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "session_index" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "screen_view_in_session_index" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "screen_views_in_session" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "dvce_created_tstamp" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "collector_tstamp" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "derived_tstamp" - } - }, - - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "screen_view_name" - } - }, - { - "expectation_type": "expect_column_values_to_be_unique", - "kwargs": { - "column": "event_id" - } - }, - { - "expectation_type": "expect_column_values_to_be_unique", - "kwargs": { - "column": "screen_view_id" - } - } - ], - "meta": { - "versions": { - "test_suite_version": "1.0.1", - "redshift_model_version": "1.0.0" - }, - "great_expectations.__version__": "0.12.0" - } -} diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_sessions.json b/.test/great_expectations/expectations/mobile/v1/mobile_sessions.json index 4b19de13..029bc2ac 100644 --- a/.test/great_expectations/expectations/mobile/v1/mobile_sessions.json +++ b/.test/great_expectations/expectations/mobile/v1/mobile_sessions.json @@ -154,7 +154,8 @@ ], "meta": { "versions": { - "test_suite_version": "1.0.1", + "test_suite_version": "1.0.2", + "redshift_model_version": "1.1.0", "bigquery_model_version": "1.0.0", "snowflake_model_version": "1.0.0" }, diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_sessions_redshift.json b/.test/great_expectations/expectations/mobile/v1/mobile_sessions_redshift.json deleted file mode 100644 index 6ef2fbcd..00000000 --- a/.test/great_expectations/expectations/mobile/v1/mobile_sessions_redshift.json +++ /dev/null @@ -1,154 +0,0 @@ -{ - "data_asset_type": "Dataset", - "expectation_suite_name": "mobile_sessions", - "expectations": [ - { - "expectation_type": "expect_table_column_count_to_equal", - "kwargs": { - "value": 62 - } - }, - { - "expectation_type": "expect_table_columns_to_match_ordered_list", - "kwargs": { - "column_list": [ - "app_id", - "session_id", - "session_index", - "previous_session_id", - "session_first_event_id", - "session_last_event_id", - "start_tstamp", - "end_tstamp", - "user_id", - "device_user_id", - "network_userid", - "session_duration_s", - "has_install", - "screen_views", - "screen_names_viewed", - "app_errors", - "fatal_app_errors", - "first_event_name", - "last_event_name", - "first_screen_view_name", - "first_screen_view_transition_type", - "first_screen_view_type", - "last_screen_view_name", - "last_screen_view_transition_type", - "last_screen_view_type", - "platform", - "dvce_screenwidth", - "dvce_screenheight", - "device_manufacturer", - "device_model", - "os_type", - "os_version", - "android_idfa", - "apple_idfa", - "apple_idfv", - "open_idfa", - "device_latitude", - "device_longitude", - "device_latitude_longitude_accuracy", - "device_altitude", - "device_altitude_accuracy", - "device_bearing", - "device_speed", - "geo_country", - "geo_region", - "geo_city", - "geo_zipcode", - "geo_latitude", - "geo_longitude", - "geo_region_name", - "geo_timezone", - "user_ipaddress", - "useragent", - "name_tracker", - "v_tracker", - "carrier", - "network_technology", - "network_type", - "first_build", - "last_build", - "first_version", - "last_version" - ] - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "session_id" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "session_index" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "session_first_event_id" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "session_last_event_id" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "start_tstamp" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "end_tstamp" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "device_user_id" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "network_userid" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "first_event_name" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "last_event_name" - } - }, - { - "expectation_type": "expect_column_values_to_be_unique", - "kwargs": { - "column": "session_id" - } - } - ], - "meta": { - "versions": { - "test_suite_version": "1.0.1", - "redshift_model_version": "1.0.0" - }, - "great_expectations.__version__": "0.12.0" - } -} diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_staging_reconciliation.json b/.test/great_expectations/expectations/mobile/v1/mobile_staging_reconciliation.json index 641bf7f1..a482a484 100644 --- a/.test/great_expectations/expectations/mobile/v1/mobile_staging_reconciliation.json +++ b/.test/great_expectations/expectations/mobile/v1/mobile_staging_reconciliation.json @@ -87,8 +87,8 @@ "meta": { "great_expectations.__version__": "0.12.0", "versions": { - "test_suite_version": "1.0.1", - "redshift_model_version": "1.0.0", + "test_suite_version": "1.0.2", + "redshift_model_version": "1.1.0", "bigquery_model_version": "1.0.0", "snowflake_model_version": "1.0.0" } diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_users.json b/.test/great_expectations/expectations/mobile/v1/mobile_users.json index 34eafcf2..db22e906 100644 --- a/.test/great_expectations/expectations/mobile/v1/mobile_users.json +++ b/.test/great_expectations/expectations/mobile/v1/mobile_users.json @@ -113,7 +113,8 @@ ], "meta": { "versions": { - "test_suite_version": "1.0.1", + "test_suite_version": "1.0.2", + "redshift_model_version": "1.1.0", "bigquery_model_version": "1.0.0", "snowflake_model_version": "1.0.0" }, diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_users_redshift.json b/.test/great_expectations/expectations/mobile/v1/mobile_users_redshift.json deleted file mode 100644 index 5b188f6d..00000000 --- a/.test/great_expectations/expectations/mobile/v1/mobile_users_redshift.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "data_asset_type": "Dataset", - "expectation_suite_name": "mobile_users", - "expectations": [ - { - "expectation_type": "expect_table_column_count_to_equal", - "kwargs": { - "value": 40 - } - }, - { - "expectation_type": "expect_table_columns_to_match_ordered_list", - "kwargs": { - "column_list": [ - "user_id", - "device_user_id", - "network_userid", - "start_tstamp", - "end_tstamp", - "screen_views", - "screen_names_viewed", - "sessions", - "sessions_duration_s", - "active_days", - "app_errors", - "fatal_app_errors", - "first_screen_view_name", - "first_screen_view_transition_type", - "first_screen_view_type", - "last_screen_view_name", - "last_screen_view_transition_type", - "last_screen_view_type", - "platform", - "dvce_screenwidth", - "dvce_screenheight", - "device_manufacturer", - "device_model", - "os_type", - "first_os_version", - "last_os_version", - "android_idfa", - "apple_idfa", - "apple_idfv", - "open_idfa", - "geo_country", - "geo_region", - "geo_city", - "geo_zipcode", - "geo_latitude", - "geo_longitude", - "geo_region_name", - "geo_timezone", - "first_carrier", - "last_carrier" - ] - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "device_user_id" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "network_userid" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "start_tstamp" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "end_tstamp" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "sessions" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "sessions_duration_s" - } - }, - { - "expectation_type": "expect_column_values_to_not_be_null", - "kwargs": { - "column": "active_days" - } - }, - { - "expectation_type": "expect_column_values_to_be_unique", - "kwargs": { - "column": "device_user_id" - } - } - ], - "meta": { - "versions": { - "test_suite_version": "1.0.1", - "redshift_model_version": "1.0.0" - }, - "great_expectations.__version__": "0.12.0" - } -} diff --git a/.test/great_expectations/validation_configs/mobile/v1/redshift/perm_tables.json b/.test/great_expectations/validation_configs/mobile/v1/redshift/perm_tables.json index ea27dd3f..7bdb7e2f 100644 --- a/.test/great_expectations/validation_configs/mobile/v1/redshift/perm_tables.json +++ b/.test/great_expectations/validation_configs/mobile/v1/redshift/perm_tables.json @@ -7,7 +7,7 @@ "table": "mobile_screen_views", "datasource": "redshift" }, - "expectation_suite_names": ["mobile.v1.mobile_screen_views_redshift"] + "expectation_suite_names": ["mobile.v1.mobile_screen_views"] }, { "batch_kwargs": { @@ -22,7 +22,7 @@ "table": "mobile_sessions", "datasource": "redshift" }, - "expectation_suite_names": ["mobile.v1.mobile_sessions_redshift"] + "expectation_suite_names": ["mobile.v1.mobile_sessions"] }, { "batch_kwargs": { @@ -30,7 +30,7 @@ "table": "mobile_users", "datasource": "redshift" }, - "expectation_suite_names": ["mobile.v1.mobile_users_redshift"] + "expectation_suite_names": ["mobile.v1.mobile_users"] }, { "batch_kwargs": { diff --git a/.test/great_expectations/validation_configs/mobile/v1/redshift/temp_tables.json b/.test/great_expectations/validation_configs/mobile/v1/redshift/temp_tables.json index c22b26c8..25f9a3e8 100644 --- a/.test/great_expectations/validation_configs/mobile/v1/redshift/temp_tables.json +++ b/.test/great_expectations/validation_configs/mobile/v1/redshift/temp_tables.json @@ -23,7 +23,7 @@ "table": "mobile_screen_views_this_run", "datasource": "redshift" }, - "expectation_suite_names": ["mobile.v1.mobile_screen_views_redshift"] + "expectation_suite_names": ["mobile.v1.mobile_screen_views"] }, { "batch_kwargs": { @@ -31,7 +31,7 @@ "table": "mobile_screen_views_staged", "datasource": "redshift" }, - "expectation_suite_names": ["mobile.v1.mobile_screen_views_redshift"] + "expectation_suite_names": ["mobile.v1.mobile_screen_views"] }, { "batch_kwargs": { @@ -39,7 +39,7 @@ "table": "mobile_sessions_this_run", "datasource": "redshift" }, - "expectation_suite_names": ["mobile.v1.mobile_sessions_redshift"] + "expectation_suite_names": ["mobile.v1.mobile_sessions"] }, { "batch_kwargs": { @@ -47,7 +47,7 @@ "table": "mobile_users_this_run", "datasource": "redshift" }, - "expectation_suite_names": ["mobile.v1.mobile_users_redshift"] + "expectation_suite_names": ["mobile.v1.mobile_users"] }, { "batch_kwargs": { diff --git a/CHANGELOG b/CHANGELOG index f8111b5a..34478320 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,12 @@ +Redshift Mobile Version 1.1.0 (2021-05-17) +--------------------------------------- +Redshift Mobile: Increase session_id character limit in manifest (Close #93) +Redshift Mobile: Add simplified configs (Close #89) +Redshift Mobile: Set app errors module to disabled by default (Close #88) +Redshift Mobile: Add model_tstamp to derived tables (Close #82) +Redshift Mobile: Move app errors columns to end of table (Close #83) +Redshift Mobile: Remove CTE from user-aggs (Close #80) + Snowflake Mobile Version 1.0.0 (2021-05-06) --------------------------------------- Add Snowflake mobile model v1 (Close #85) diff --git a/mobile/v1/redshift/CHANGELOG b/mobile/v1/redshift/CHANGELOG index 58cfaa67..3fa2cf2d 100644 --- a/mobile/v1/redshift/CHANGELOG +++ b/mobile/v1/redshift/CHANGELOG @@ -1,3 +1,12 @@ +Version 1.1.0 (2021-05-17) +-------------------------- +Redshift Mobile: Increase session_id character limit in manifest (Close #93) +Redshift Mobile: Add simplified configs (Close #89) +Redshift Mobile: Set app errors module to disabled by default (Close #88) +Redshift Mobile: Add model_tstamp to derived tables (Close #82) +Redshift Mobile: Move app errors columns to end of table (Close #83) +Redshift Mobile: Remove CTE from user-aggs (Close #80) + Version 1.0.0 (2021-03-25) -------------------------- Update scripts for mobile model (#66) diff --git a/mobile/v1/redshift/sql-runner/configs/datamodeling.json b/mobile/v1/redshift/sql-runner/configs/datamodeling.json index 34e65b62..cbf34b85 100644 --- a/mobile/v1/redshift/sql-runner/configs/datamodeling.json +++ b/mobile/v1/redshift/sql-runner/configs/datamodeling.json @@ -10,50 +10,58 @@ }, { "playbook": "standard/02-screen-views/01-screen-views-main", - "dependsOn": ["standard/01-base/01-base-main"] + "dependsOn": [ + "standard/01-base/01-base-main" + ] }, { "playbook": "standard/03-optional-modules/01-app-errors/01-app-errors-main", - "dependsOn": ["standard/01-base/01-base-main"] + "dependsOn": [ + "standard/01-base/01-base-main" + ] }, { "playbook": "standard/04-sessions/01-sessions-main", - "dependsOn": ["standard/01-base/01-base-main", - "standard/02-screen-views/01-screen-views-main", - "standard/03-optional-modules/01-app-errors/01-app-errors-main" - ] + "dependsOn": [ + "standard/02-screen-views/01-screen-views-main", + "standard/03-optional-modules/01-app-errors/01-app-errors-main" + ] }, { "playbook": "standard/05-users/01-users-main", - "dependsOn": ["standard/04-sessions/01-sessions-main"] + "dependsOn": [ + "standard/04-sessions/01-sessions-main" + ] }, { "playbook": "standard/01-base/99-base-complete", - "dependsOn": ["standard/01-base/01-base-main", - "standard/02-screen-views/01-screen-views-main", - "standard/03-optional-modules/01-app-errors/01-app-errors-main", - "standard/04-sessions/01-sessions-main" - ] + "dependsOn": [ + "standard/05-users/01-users-main" + ] }, { "playbook": "standard/02-screen-views/99-screen-views-complete", - "dependsOn": ["standard/02-screen-views/01-screen-views-main"] + "dependsOn": [ + "standard/05-users/01-users-main" + ] }, { "playbook": "standard/03-optional-modules/01-app-errors/99-app-errors-complete", - "dependsOn": ["standard/03-optional-modules/01-app-errors/01-app-errors-main"] + "dependsOn": [ + "standard/05-users/01-users-main" + ] }, { "playbook": "standard/04-sessions/99-sessions-complete", - "dependsOn": ["standard/02-screen-views/01-screen-views-main", - "standard/03-optional-modules/01-app-errors/01-app-errors-main", - "standard/04-sessions/01-sessions-main", - "standard/05-users/01-users-main" - ] + "dependsOn": [ + "standard/05-users/01-users-main" + ] }, { "playbook": "standard/05-users/99-users-complete", - "dependsOn": ["standard/05-users/01-users-main"] + "dependsOn": [ + "standard/05-users/01-users-main" + ] } ], "lockType": "hard", diff --git a/mobile/v1/redshift/sql-runner/configs/datamodeling_custom_module.json b/mobile/v1/redshift/sql-runner/configs/datamodeling_custom_module.json index dc4ad294..92516d67 100644 --- a/mobile/v1/redshift/sql-runner/configs/datamodeling_custom_module.json +++ b/mobile/v1/redshift/sql-runner/configs/datamodeling_custom_module.json @@ -10,59 +10,71 @@ }, { "playbook": "standard/02-screen-views/01-screen-views-main", - "dependsOn": ["standard/01-base/01-base-main"] + "dependsOn": [ + "standard/01-base/01-base-main" + ] }, { "playbook": "standard/03-optional-modules/01-app-errors/01-app-errors-main", - "dependsOn": ["standard/01-base/01-base-main"] + "dependsOn": [ + "standard/01-base/01-base-main" + ] }, { "playbook": "standard/04-sessions/01-sessions-main", - "dependsOn": ["standard/01-base/01-base-main", - "standard/02-screen-views/01-screen-views-main", - "standard/03-optional-modules/01-app-errors/01-app-errors-main" - ] + "dependsOn": [ + "standard/02-screen-views/01-screen-views-main", + "standard/03-optional-modules/01-app-errors/01-app-errors-main" + ] }, { "playbook": "custom/04-session-goals/01-session-goals-main", - "dependsOn": ["standard/04-sessions/01-sessions-main"] + "dependsOn": [ + "standard/04-sessions/01-sessions-main" + ] }, { "playbook": "standard/05-users/01-users-main", - "dependsOn": ["standard/04-sessions/01-sessions-main"] + "dependsOn": [ + "standard/04-sessions/01-sessions-main" + ] }, { "playbook": "standard/01-base/99-base-complete", - "dependsOn": ["standard/01-base/01-base-main", - "standard/02-screen-views/01-screen-views-main", - "standard/03-optional-modules/01-app-errors/01-app-errors-main", - "standard/04-sessions/01-sessions-main" - ] + "dependsOn": [ + "standard/05-users/01-users-main" + ] }, { "playbook": "standard/02-screen-views/99-screen-views-complete", - "dependsOn": ["standard/02-screen-views/01-screen-views-main"] + "dependsOn": [ + "standard/05-users/01-users-main" + ] }, { "playbook": "standard/03-optional-modules/01-app-errors/99-app-errors-complete", - "dependsOn": ["standard/03-optional-modules/01-app-errors/01-app-errors-main"] + "dependsOn": [ + "standard/05-users/01-users-main" + ] }, { "playbook": "custom/04-session-goals/99-session-goals-complete", - "dependsOn": ["custom/04-session-goals/01-session-goals-main"] + "dependsOn": [ + "custom/04-session-goals/01-session-goals-main" + ] }, { "playbook": "standard/04-sessions/99-sessions-complete", - "dependsOn": ["standard/02-screen-views/01-screen-views-main", - "standard/03-optional-modules/01-app-errors/01-app-errors-main", - "standard/04-sessions/01-sessions-main", - "custom/04-session-goals/99-session-goals-complete", - "standard/05-users/01-users-main" - ] + "dependsOn": [ + "custom/04-session-goals/01-session-goals-main", + "standard/05-users/01-users-main" + ] }, { "playbook": "standard/05-users/99-users-complete", - "dependsOn": ["standard/05-users/01-users-main"] + "dependsOn": [ + "standard/05-users/01-users-main" + ] } ], "lockType": "hard", diff --git a/mobile/v1/redshift/sql-runner/configs/pre_test.json b/mobile/v1/redshift/sql-runner/configs/pre_test.json index 5e7691a1..55e17ebb 100644 --- a/mobile/v1/redshift/sql-runner/configs/pre_test.json +++ b/mobile/v1/redshift/sql-runner/configs/pre_test.json @@ -10,30 +10,34 @@ }, { "playbook": "standard/02-screen-views/01-screen-views-main", - "dependsOn": ["standard/01-base/01-base-main"] + "dependsOn": [ + "standard/01-base/01-base-main" + ] }, { "playbook": "standard/03-optional-modules/01-app-errors/01-app-errors-main", - "dependsOn": ["standard/01-base/01-base-main"] + "dependsOn": [ + "standard/01-base/01-base-main" + ] }, { "playbook": "standard/04-sessions/01-sessions-main", - "dependsOn": ["standard/01-base/01-base-main", - "standard/02-screen-views/01-screen-views-main", - "standard/03-optional-modules/01-app-errors/01-app-errors-main" - ] + "dependsOn": [ + "standard/02-screen-views/01-screen-views-main", + "standard/03-optional-modules/01-app-errors/01-app-errors-main" + ] }, { "playbook": "standard/05-users/01-users-main", - "dependsOn": ["standard/04-sessions/01-sessions-main"] + "dependsOn": [ + "standard/04-sessions/01-sessions-main" + ] }, { "playbook": "tests/00-staging-reconciliation/01-staging-reconciliation-main", - "dependsOn": ["standard/01-base/01-base-main", - "standard/02-screen-views/01-screen-views-main", - "standard/03-optional-modules/01-app-errors/01-app-errors-main", - "standard/04-sessions/01-sessions-main" - ] + "dependsOn": [ + "standard/04-sessions/01-sessions-main" + ] } ], "lockType": "hard", diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl index 6908b9ae..99529555 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :model: mobile :scratch_schema: scratch :output_schema: derived diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl index 8a6af119..51f17f38 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :model: mobile :scratch_schema: scratch :output_schema: derived diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl index 98fdf3f7..bbc19465 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :model: mobile :scratch_schema: scratch :output_schema: derived diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl index e231ec47..b769f4d1 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl @@ -8,24 +8,24 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 - :model: mobile - :input_schema: atomic - :scratch_schema: scratch - :output_schema: derived - :entropy: "" - :stage_next: true - :start_date: 2020-01-01 + :model_version: redshift/mobile/1.1.0 + :model: mobile + :input_schema: atomic + :scratch_schema: scratch + :output_schema: derived + :entropy: "" + :stage_next: true + :start_date: 2020-01-01 :lookback_window_hours: :days_late_allowed: :update_cadence_days: :session_lookback_days: - :mobile_context: false - :geolocation_context: false - :application_context: false - :screen_context: false - :platform_filters: [] - :app_id_filters: [] + :mobile_context: false + :geolocation_context: false + :application_context: false + :screen_context: false + :platform_filters: [] + :app_id_filters: [] :steps: - :name: 00-setup-base :queries: diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl index 09ad826d..217736aa 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :model: mobile :scratch_schema: scratch :output_schema: derived diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl index 945dc0d9..b7b9481b 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :model: mobile :scratch_schema: scratch :output_schema: derived diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/01-screen-views-main.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/01-screen-views-main.yml.tmpl index d088ad50..096f3f78 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/01-screen-views-main.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/01-screen-views-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :input_schema: atomic :scratch_schema: scratch :output_schema: derived diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/99-screen-views-complete.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/99-screen-views-complete.yml.tmpl index 169a34a7..08c1dfce 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/99-screen-views-complete.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/99-screen-views-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/XX-destroy-screen-views.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/XX-destroy-screen-views.yml.tmpl index c2c6804e..467977e1 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/XX-destroy-screen-views.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/02-screen-views/XX-destroy-screen-views.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/01-app-errors-main.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/01-app-errors-main.yml.tmpl index 601fbb6f..7a7d2b0f 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/01-app-errors-main.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/01-app-errors-main.yml.tmpl @@ -8,11 +8,11 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :input_schema: atomic :scratch_schema: scratch :output_schema: derived - :enabled: true + :enabled: false :entropy: "" :upsert_lookback_days: :skip_derived: diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/99-app-errors-complete.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/99-app-errors-complete.yml.tmpl index a0eafb1e..0b856a1d 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/99-app-errors-complete.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/99-app-errors-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/XX-destroy-app-errors.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/XX-destroy-app-errors.yml.tmpl index 0699a13c..2bb21f7b 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/XX-destroy-app-errors.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/03-optional-modules/01-app-errors/XX-destroy-app-errors.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/01-sessions-main.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/01-sessions-main.yml.tmpl index 8ac9e82d..f1f2b865 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/01-sessions-main.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/01-sessions-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/99-sessions-complete.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/99-sessions-complete.yml.tmpl index 7fd43669..62bfff1f 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/99-sessions-complete.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/99-sessions-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/XX-destroy-sessions.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/XX-destroy-sessions.yml.tmpl index c44a7aa9..a63a7b98 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/XX-destroy-sessions.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/04-sessions/XX-destroy-sessions.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/01-users-main.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/01-users-main.yml.tmpl index 8f0f62f3..3cad648f 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/01-users-main.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/01-users-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/99-users-complete.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/99-users-complete.yml.tmpl index f7d1332a..54cb7e2d 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/99-users-complete.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/99-users-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/XX-destroy-users.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/XX-destroy-users.yml.tmpl index 80cb5ea1..d38798c1 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/XX-destroy-users.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/standard/05-users/XX-destroy-users.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/playbooks/tests/00-staging-reconciliation/01-staging-reconciliation-main.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/tests/00-staging-reconciliation/01-staging-reconciliation-main.yml.tmpl index b73c9816..0097a855 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/tests/00-staging-reconciliation/01-staging-reconciliation-main.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/tests/00-staging-reconciliation/01-staging-reconciliation-main.yml.tmpl @@ -8,11 +8,11 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :model: mobile :scratch_schema: scratch :entropy: "" - :app_errors: + :app_errors: false :steps: - :name: 00-staging-reconciliation :queries: diff --git a/mobile/v1/redshift/sql-runner/playbooks/tests/00-staging-reconciliation/99-staging-reconciliation-complete.yml.tmpl b/mobile/v1/redshift/sql-runner/playbooks/tests/00-staging-reconciliation/99-staging-reconciliation-complete.yml.tmpl index 56a851b2..7ce6a41c 100644 --- a/mobile/v1/redshift/sql-runner/playbooks/tests/00-staging-reconciliation/99-staging-reconciliation-complete.yml.tmpl +++ b/mobile/v1/redshift/sql-runner/playbooks/tests/00-staging-reconciliation/99-staging-reconciliation-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/mobile/1.0.0 + :model_version: redshift/mobile/1.1.0 :model: mobile :scratch_schema: scratch :entropy: "" diff --git a/mobile/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql b/mobile/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql index 6db02916..bae943e3 100644 --- a/mobile/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql +++ b/mobile/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql @@ -110,7 +110,7 @@ INSERT INTO {{.output_schema}}.{{.model}}_base_event_id_manifest{{.entropy}} ( ); CREATE TABLE IF NOT EXISTS {{.output_schema}}.{{.model}}_base_session_id_manifest{{.entropy}} ( - session_id VARCHAR(36), + session_id VARCHAR(128), min_tstamp TIMESTAMP ) DISTSTYLE KEY @@ -119,7 +119,7 @@ SORTKEY (min_tstamp); INSERT INTO {{.output_schema}}.{{.model}}_base_session_id_manifest{{.entropy}} ( SELECT - 'seed'::VARCHAR(36), + 'seed'::VARCHAR(128), '{{.start_date}}'::TIMESTAMP WHERE diff --git a/mobile/v1/redshift/sql-runner/sql/standard/02-screen-views/01-main/00-setup-screen-views.sql b/mobile/v1/redshift/sql-runner/sql/standard/02-screen-views/01-main/00-setup-screen-views.sql index 7d0470b7..81fb5ffd 100644 --- a/mobile/v1/redshift/sql-runner/sql/standard/02-screen-views/01-main/00-setup-screen-views.sql +++ b/mobile/v1/redshift/sql-runner/sql/standard/02-screen-views/01-main/00-setup-screen-views.sql @@ -103,6 +103,7 @@ CREATE TABLE IF NOT EXISTS {{.output_schema}}.mobile_screen_views{{.entropy}} ( dvce_created_tstamp TIMESTAMP ENCODE ZSTD, collector_tstamp TIMESTAMP ENCODE ZSTD, derived_tstamp TIMESTAMP ENCODE RAW, --raw for sort key + model_tstamp TIMESTAMP ENCODE ZSTD, screen_view_name VARCHAR(4096) ENCODE ZSTD, screen_view_transition_type VARCHAR(4096) ENCODE ZSTD, screen_view_type VARCHAR(4096) ENCODE ZSTD, diff --git a/mobile/v1/redshift/sql-runner/sql/standard/02-screen-views/01-main/04-screen-views.sql b/mobile/v1/redshift/sql-runner/sql/standard/02-screen-views/01-main/04-screen-views.sql index 209ba73e..56a51a2d 100644 --- a/mobile/v1/redshift/sql-runner/sql/standard/02-screen-views/01-main/04-screen-views.sql +++ b/mobile/v1/redshift/sql-runner/sql/standard/02-screen-views/01-main/04-screen-views.sql @@ -58,6 +58,7 @@ AS( ev.dvce_created_tstamp, ev.collector_tstamp, ev.derived_tstamp, + CURRENT_TIMESTAMP AS model_tstamp, ev.screen_view_name, ev.screen_view_transition_type, diff --git a/mobile/v1/redshift/sql-runner/sql/standard/03-optional-modules/01-app-errors/01-main/00-setup-app-errors.sql b/mobile/v1/redshift/sql-runner/sql/standard/03-optional-modules/01-app-errors/01-main/00-setup-app-errors.sql index 185103cf..d896630a 100644 --- a/mobile/v1/redshift/sql-runner/sql/standard/03-optional-modules/01-app-errors/01-main/00-setup-app-errors.sql +++ b/mobile/v1/redshift/sql-runner/sql/standard/03-optional-modules/01-app-errors/01-main/00-setup-app-errors.sql @@ -106,15 +106,7 @@ CREATE TABLE IF NOT EXISTS {{.scratch_schema}}.mobile_app_errors_staged{{.entrop dvce_created_tstamp TIMESTAMP ENCODE ZSTD, collector_tstamp TIMESTAMP ENCODE ZSTD, derived_tstamp TIMESTAMP ENCODE RAW, --raw for sort key - message VARCHAR(2048) ENCODE ZSTD, - programming_language VARCHAR(12) ENCODE ZSTD, - class_name VARCHAR(1024) ENCODE ZSTD, - exception_name VARCHAR(1024) ENCODE ZSTD, - is_fatal BOOLEAN ENCODE ZSTD, - line_number INT ENCODE ZSTD, - stack_trace VARCHAR(8192) ENCODE ZSTD, - thread_id INT ENCODE ZSTD, - thread_name VARCHAR(1024) ENCODE ZSTD, + model_tstamp TIMESTAMP ENCODE ZSTD, platform VARCHAR(255) ENCODE ZSTD, dvce_screenwidth INT ENCODE ZSTD, dvce_screenheight INT ENCODE ZSTD, @@ -155,7 +147,16 @@ CREATE TABLE IF NOT EXISTS {{.scratch_schema}}.mobile_app_errors_staged{{.entrop network_type VARCHAR(7) ENCODE ZSTD, build VARCHAR(255) ENCODE ZSTD, version VARCHAR(255) ENCODE ZSTD, - event_index_in_session INT ENCODE ZSTD + event_index_in_session INT ENCODE ZSTD, + message VARCHAR(2048) ENCODE ZSTD, + programming_language VARCHAR(12) ENCODE ZSTD, + class_name VARCHAR(1024) ENCODE ZSTD, + exception_name VARCHAR(1024) ENCODE ZSTD, + is_fatal BOOLEAN ENCODE ZSTD, + line_number INT ENCODE ZSTD, + stack_trace VARCHAR(8192) ENCODE ZSTD, + thread_id INT ENCODE ZSTD, + thread_name VARCHAR(1024) ENCODE ZSTD ) DISTSTYLE KEY DISTKEY (event_id) diff --git a/mobile/v1/redshift/sql-runner/sql/standard/03-optional-modules/01-app-errors/01-main/03-app-errors.sql b/mobile/v1/redshift/sql-runner/sql/standard/03-optional-modules/01-app-errors/01-main/03-app-errors.sql index 26cda952..65ec951d 100644 --- a/mobile/v1/redshift/sql-runner/sql/standard/03-optional-modules/01-app-errors/01-main/03-app-errors.sql +++ b/mobile/v1/redshift/sql-runner/sql/standard/03-optional-modules/01-app-errors/01-main/03-app-errors.sql @@ -41,17 +41,7 @@ es.dvce_created_tstamp, es.collector_tstamp, es.derived_tstamp, - - --Error contexts - ae.message, - ae.programming_language, - ae.class_name, - ae.exception_name, - ae.is_fatal, - ae.line_number, - ae.stack_trace, - ae.thread_id, - ae.thread_name, + CURRENT_TIMESTAMP AS model_tstamp, es.platform, es.dvce_screenwidth, @@ -98,7 +88,18 @@ es.build, es.version, - es.event_index_in_session + es.event_index_in_session, + + --Error details + ae.message, + ae.programming_language, + ae.class_name, + ae.exception_name, + ae.is_fatal, + ae.line_number, + ae.stack_trace, + ae.thread_id, + ae.thread_name FROM {{.scratch_schema}}.mobile_events_staged{{.entropy}} es diff --git a/mobile/v1/redshift/sql-runner/sql/standard/04-sessions/01-main/00-setup-sessions.sql b/mobile/v1/redshift/sql-runner/sql/standard/04-sessions/01-main/00-setup-sessions.sql index bba3500b..f1e3153a 100644 --- a/mobile/v1/redshift/sql-runner/sql/standard/04-sessions/01-main/00-setup-sessions.sql +++ b/mobile/v1/redshift/sql-runner/sql/standard/04-sessions/01-main/00-setup-sessions.sql @@ -96,6 +96,7 @@ CREATE TABLE IF NOT EXISTS {{.output_schema}}.mobile_sessions{{.entropy}} ( session_last_event_id CHAR(36) ENCODE ZSTD, start_tstamp TIMESTAMP ENCODE RAW, --raw for sk end_tstamp TIMESTAMP ENCODE ZSTD, + model_tstamp TIMESTAMP ENCODE ZSTD, user_id VARCHAR(255) ENCODE ZSTD, device_user_id VARCHAR(4096) ENCODE ZSTD, network_userid VARCHAR(128) ENCODE ZSTD, diff --git a/mobile/v1/redshift/sql-runner/sql/standard/04-sessions/01-main/03-sessions.sql b/mobile/v1/redshift/sql-runner/sql/standard/04-sessions/01-main/03-sessions.sql index d6c3b42b..0d381567 100644 --- a/mobile/v1/redshift/sql-runner/sql/standard/04-sessions/01-main/03-sessions.sql +++ b/mobile/v1/redshift/sql-runner/sql/standard/04-sessions/01-main/03-sessions.sql @@ -33,6 +33,7 @@ AS ( sa.start_tstamp, sa.end_tstamp, + CURRENT_TIMESTAMP AS model_tstamp, es.user_id, es.device_user_id, diff --git a/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/00-setup-users.sql b/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/00-setup-users.sql index b0f14d4f..13a9f289 100644 --- a/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/00-setup-users.sql +++ b/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/00-setup-users.sql @@ -112,6 +112,7 @@ CREATE TABLE IF NOT EXISTS {{.output_schema}}.mobile_users{{.entropy}} ( start_tstamp TIMESTAMP ENCODE RAW, --raw for sk end_tstamp TIMESTAMP ENCODE ZSTD, + model_tstamp TIMESTAMP ENCODE ZSTD, screen_views INT ENCODE ZSTD, screen_names_viewed INT ENCODE ZSTD, diff --git a/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/04-users-aggs.sql b/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/04-users-aggs.sql index 6ea9f5ca..36bd9525 100644 --- a/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/04-users-aggs.sql +++ b/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/04-users-aggs.sql @@ -21,42 +21,22 @@ CREATE TABLE {{.scratch_schema}}.mobile_users_aggregates{{.entropy}} DISTKEY (device_user_id) SORTKEY (device_user_id) AS( - WITH prep AS ( - SELECT - device_user_id, - -- time - MIN(start_tstamp) AS start_tstamp, - MAX(end_tstamp) AS end_tstamp, - -- engagement - SUM(screen_views) AS screen_views, - SUM(screen_names_viewed) AS screen_names_viewed, - COUNT(DISTINCT session_id) AS sessions, - SUM(session_duration_s) AS sessions_duration_s, - COUNT(DISTINCT DATE_TRUNC('d', start_tstamp)) AS active_days, - --errors - SUM(app_errors) AS app_errors, - SUM(fatal_app_errors) AS fatal_app_errors - - FROM {{.scratch_schema}}.mobile_users_sessions_this_run{{.entropy}} - - GROUP BY 1 - ) - SELECT device_user_id, -- time - start_tstamp, - end_tstamp, + MIN(start_tstamp) AS start_tstamp, + MAX(end_tstamp) AS end_tstamp, -- engagement - screen_views, - screen_names_viewed, - sessions, - sessions_duration_s, - active_days, + SUM(screen_views) AS screen_views, + SUM(screen_names_viewed) AS screen_names_viewed, + COUNT(DISTINCT session_id) AS sessions, + SUM(session_duration_s) AS sessions_duration_s, + COUNT(DISTINCT DATE_TRUNC('d', start_tstamp)) AS active_days, --errors - app_errors, - fatal_app_errors + SUM(app_errors) AS app_errors, + SUM(fatal_app_errors) AS fatal_app_errors - FROM prep + FROM {{.scratch_schema}}.mobile_users_sessions_this_run{{.entropy}} + GROUP BY 1 ); diff --git a/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/06-users.sql b/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/06-users.sql index 0d057513..878a70f4 100644 --- a/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/06-users.sql +++ b/mobile/v1/redshift/sql-runner/sql/standard/05-users/01-main/06-users.sql @@ -29,6 +29,7 @@ AS ( b.start_tstamp, b.end_tstamp, + CURRENT_TIMESTAMP AS model_tstamp, -- engagement fields b.screen_views,