From 090017008b63cf7da94eac6e1d3c87c03eb59814 Mon Sep 17 00:00:00 2001 From: Oguzhan Unlu Date: Thu, 12 Sep 2024 17:01:13 +0300 Subject: [PATCH 1/3] Upgrade schema-ddl to 0.25.0-M1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f9233a9..d6e12f0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -38,7 +38,7 @@ object Dependencies { val azureSdk = "1.11.4" // Snowplow - val schemaDdl = "0.23.0" + val schemaDdl = "0.25.0-M1" val badrows = "2.3.0" val igluClient = "3.2.0" val tracker = "2.0.0" From bb288a095f47e6a0bf4133fb05f6390411ba0ee4 Mon Sep 17 00:00:00 2001 From: Oguzhan Unlu Date: Fri, 13 Sep 2024 17:41:19 +0300 Subject: [PATCH 2/3] Add field starting with digit test --- .../schemas/myvendor/digit/jsonschema/1-0-0 | 14 ++++++++ .../transform/NonAtomicFieldsSpec.scala | 32 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 modules/loaders-common/src/test/resources/iglu-client-embedded/schemas/myvendor/digit/jsonschema/1-0-0 diff --git a/modules/loaders-common/src/test/resources/iglu-client-embedded/schemas/myvendor/digit/jsonschema/1-0-0 b/modules/loaders-common/src/test/resources/iglu-client-embedded/schemas/myvendor/digit/jsonschema/1-0-0 new file mode 100644 index 0000000..2610e5f --- /dev/null +++ b/modules/loaders-common/src/test/resources/iglu-client-embedded/schemas/myvendor/digit/jsonschema/1-0-0 @@ -0,0 +1,14 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "self": { + "vendor": "myvendor", + "name": "digit", + "format": "jsonschema", + "version": "1-0-0" + }, + "type": "object", + "properties": { + "1col_a": {"type": "string"} + }, + "required": ["1col_a"] +} diff --git a/modules/loaders-common/src/test/scala/com.snowplowanalytics.snowplow.loaders/transform/NonAtomicFieldsSpec.scala b/modules/loaders-common/src/test/scala/com.snowplowanalytics.snowplow.loaders/transform/NonAtomicFieldsSpec.scala index eda10d3..062fd4d 100644 --- a/modules/loaders-common/src/test/scala/com.snowplowanalytics.snowplow.loaders/transform/NonAtomicFieldsSpec.scala +++ b/modules/loaders-common/src/test/scala/com.snowplowanalytics.snowplow.loaders/transform/NonAtomicFieldsSpec.scala @@ -28,6 +28,7 @@ class NonAtomicFieldsSpec extends Specification with CatsEffect { return a merged schema if the batch uses all schemas in a series $ue3 return nothing for the Iglu Central ad_break_end_event schema $ue4 return a JSON field for the Iglu Central anything-a schema $ue5 + return a field prefixed with underscore if field starts with a digit $ueDigit when resolving for known schemas in contexts should return an un-merged schema if the batch uses the first schema in a series $c1 @@ -197,6 +198,37 @@ class NonAtomicFieldsSpec extends Specification with CatsEffect { } + def ueDigit = { + val tabledEntity = TabledEntity(TabledEntity.UnstructEvent, "myvendor", "digit", 1) + + val input = Map( + tabledEntity -> Set((0, 0)) + ) + + val expected = { + val expectedStruct = Type.Struct( + NonEmptyVector.of( + Field("_1col_a", Type.String, Required).copy(accessors = Set("1col_a")) + ) + ) + + val expectedField = Field("unstruct_event_myvendor_digit_1", expectedStruct, Nullable, Set.empty) + + TypedTabledEntity( + tabledEntity, + expectedField, + Set((0, 0)), + Nil + ) + } + + NonAtomicFields.resolveTypes(embeddedResolver, input, List.empty).map { case NonAtomicFields.Result(fields, failures) => + (failures must beEmpty) and + (fields must haveSize(1)) and + (fields.head must beEqualTo(expected)) + } + } + def c1 = { val tabledEntity = TabledEntity(TabledEntity.Context, "myvendor", "myschema", 7) From bcabe79d392351552c257b60fd797bd06b88a796 Mon Sep 17 00:00:00 2001 From: Oguzhan Unlu Date: Fri, 13 Sep 2024 17:56:01 +0300 Subject: [PATCH 3/3] add more tests --- .../transform/TransformStructuredSpec.scala | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/modules/loaders-common/src/test/scala/com.snowplowanalytics.snowplow.loaders/transform/TransformStructuredSpec.scala b/modules/loaders-common/src/test/scala/com.snowplowanalytics.snowplow.loaders/transform/TransformStructuredSpec.scala index 78b04ba..431a78c 100644 --- a/modules/loaders-common/src/test/scala/com.snowplowanalytics.snowplow.loaders/transform/TransformStructuredSpec.scala +++ b/modules/loaders-common/src/test/scala/com.snowplowanalytics.snowplow.loaders/transform/TransformStructuredSpec.scala @@ -33,6 +33,11 @@ class TransformStructuredSpec extends Specification { Field("my_string", Type.String, Required) ) + private val simpleOneFieldStartingWithDigitSchema = + NonEmptyVector.of( + Field("1my_string", Type.String, Required) + ) + private val schemaWithAllPossibleTypes = NonEmptyVector.of( Field("my_string", Type.String, Required), @@ -71,7 +76,9 @@ class TransformStructuredSpec extends Specification { Successful transformation: Valid event with only atomic fields (no custom entities) $onlyAtomic Valid event with one custom context $oneContext + Valid event with one custom context starting with digit $oneContextDigit Valid unstruct event $unstruct + Valid unstruct event with field starting with digit $unstructDigit Valid event with two custom contexts, same major version $twoContextsSameMajor Valid event with two custom contexts, different major version $twoContextsDifferentMajor Valid event with each different type of atomic field $onlyAtomicAllTypes @@ -120,6 +127,18 @@ class TransformStructuredSpec extends Specification { assertSuccessful(inputEvent, batchInfo, expectedAllEntities = expectedOutput) } + def unstructDigit = { + val inputEvent = + createEvent(unstruct = Some(sdj(data = json"""{ "1my_string": "abc"}""", key = "iglu:com.example/mySchema/jsonschema/1-0-0"))) + val batchInfo = Result( + fields = Vector(mySchemaUnstruct(model = 1, subVersions = Set((0, 0)), simpleOneFieldStartingWithDigitSchema)), + igluFailures = List.empty + ) + val expectedOutput = List(NamedValue(name = "unstruct_event_com_example_my_schema_1", value = json"""{ "_1my_string": "abc"}""")) + + assertSuccessful(inputEvent, batchInfo, expectedAllEntities = expectedOutput) + } + def oneContext = { val inputEvent = createEvent(contexts = List(sdj(data = json"""{ "my_string": "abc"}""", key = "iglu:com.example/mySchema/jsonschema/1-0-0"))) @@ -134,6 +153,20 @@ class TransformStructuredSpec extends Specification { assertSuccessful(inputEvent, batchInfo, expectedAllEntities = expectedOutput) } + def oneContextDigit = { + val inputEvent = + createEvent(contexts = List(sdj(data = json"""{ "1my_string": "abc"}""", key = "iglu:com.example/mySchema/jsonschema/1-0-0"))) + val batchInfo = Result( + fields = Vector(mySchemaContexts(model = 1, subVersions = Set((0, 0)), simpleOneFieldStartingWithDigitSchema)), + igluFailures = List.empty + ) + val expectedOutput = List( + NamedValue(name = "contexts_com_example_my_schema_1", value = json"""[{ "_schema_version": "1-0-0", "_1my_string": "abc"}]""") + ) + + assertSuccessful(inputEvent, batchInfo, expectedAllEntities = expectedOutput) + } + def twoContextsDifferentMajor = { val inputEvent = createEvent(contexts = List(