From 9da46bbc882c11e72f95fc4dbe89341c23712f81 Mon Sep 17 00:00:00 2001 From: Johnny McQuade Date: Thu, 2 May 2024 16:31:56 +0100 Subject: [PATCH 1/6] fix: data items are sorted according to original order by default --- .../data-items/data-items.component.ts | 10 ++++++---- .../dynamic-data/dynamic-data.service.spec.ts | 7 +++++++ .../dynamic-data/dynamic-data.service.ts | 20 ++++++++++++++----- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/app/shared/components/template/components/data-items/data-items.component.ts b/src/app/shared/components/template/components/data-items/data-items.component.ts index b32c9275ee..7888ca3063 100644 --- a/src/app/shared/components/template/components/data-items/data-items.component.ts +++ b/src/app/shared/components/template/components/data-items/data-items.component.ts @@ -64,7 +64,10 @@ export class TmplDataItemsComponent extends TemplateBaseComponent implements OnD await this.dynamicDataService.ready(); const query = await this.dynamicDataService.query$("data_list", this.dataListName); this.dataQuery$ = query.pipe(debounceTime(50)).subscribe(async (data) => { - await this.renderItems(data, this._row.rows, this.parameterList); + // By default, sort the items into the order they appeared in the original data list. + // Can be overridden with a `sort` operator applied in the data-items component params + const sortedData = data.sort((a, b) => a.index_original - b.index_original); + await this.renderItems(sortedData, this._row.rows, this.parameterList); }); } else { await this.renderItems([], [], {}); @@ -185,9 +188,8 @@ export class TmplDataItemsComponent extends TemplateBaseComponent implements OnD parsed[listKey] = listValue; for (const [itemKey, itemValue] of Object.entries(listValue)) { if (typeof itemValue === "string") { - parsed[listKey][itemKey] = await this.templateVariablesService.evaluateConditionString( - itemValue - ); + parsed[listKey][itemKey] = + await this.templateVariablesService.evaluateConditionString(itemValue); } } } diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts index 50ec015cc2..6b58b792cd 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts @@ -143,6 +143,13 @@ describe("DynamicDataService", () => { expect(data[1].string).toEqual("sets an item correctly for a given _index"); }); + it("adds metadata (original index) to docs", async () => { + const obs = await service.query$("data_list", "test_flow"); + const data = await firstValueFrom(obs); + expect(data[0].index_original).toEqual(0); + expect(data[1].index_original).toEqual(1); + }); + // QA it("prevents query of non-existent data lists", async () => { let errMsg: string; diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.ts index d54b2b5c48..50edb79a5e 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.ts @@ -12,7 +12,7 @@ import { ReactiveMemoryAdapater, REACTIVE_SCHEMA_BASE } from "./adapters/reactiv import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry"; import { TopLevelProperty } from "rxdb/dist/types/types"; -type IDocWithID = { id: string }; +type IDocWithID = { [key: string]: any; id: string }; @Injectable({ providedIn: "root" }) /** @@ -178,9 +178,17 @@ export class DynamicDataService extends AsyncServiceBase { if (initialData.length === 0) { throw new Error(`No data exists for collection [${flow_name}], cannot initialise`); } - const schema = this.inferSchema(initialData[0]); + // add index property to each element before insert, for sorting queried data by original order + const initialDataWithMeta = initialData.map((el) => { + return { + ...el, + index_original: initialData.indexOf(el), + }; + }); + + const schema = this.inferSchema(initialDataWithMeta[0]); await this.db.createCollection(collectionName, schema); - await this.db.bulkInsert(collectionName, initialData); + await this.db.bulkInsert(collectionName, initialDataWithMeta); // notify any observers that collection has been created this.collectionCreators[collectionName].next(collectionName); this.collectionCreators[collectionName].complete(); @@ -191,7 +199,9 @@ export class DynamicDataService extends AsyncServiceBase { private async getInitialData(flow_type: FlowTypes.FlowType, flow_name: string) { const flowData = await this.appDataService.getSheet(flow_type, flow_name); const writeData = this.writeCache.get(flow_type, flow_name); - const writeDataArray = Object.entries(writeData || {}).map(([id, v]) => ({ ...v, id })); + const writeDataArray = Object.entries(writeData || {}).map(([id, v]) => ({ ...v, id })) as + | IDocWithID[] + | []; const mergedData = this.mergeData(flowData?.rows, writeDataArray); return mergedData; } @@ -201,7 +211,7 @@ export class DynamicDataService extends AsyncServiceBase { return `${flow_type}${flow_name}`.toLowerCase().replace(/[^a-z0-9]/g, ""); } - private mergeData(flowData: T[] = [], dbData: T[] = []) { + private mergeData(flowData: T[] = [], dbData: T[] = []) { const flowHashmap = arrayToHashmap(flowData, "id"); const dbDataHashmap = arrayToHashmap(dbData, "id"); const merged = deepMergeObjects(flowHashmap, dbDataHashmap); From 2047385223c7065f693681ed5ef1f943b59d60aa Mon Sep 17 00:00:00 2001 From: Johnny McQuade Date: Fri, 3 May 2024 12:32:03 +0100 Subject: [PATCH 2/6] chore: rename 'index_original' -> 'row_index' --- .../template/components/data-items/data-items.component.ts | 2 +- .../shared/services/dynamic-data/dynamic-data.service.spec.ts | 4 ++-- src/app/shared/services/dynamic-data/dynamic-data.service.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/app/shared/components/template/components/data-items/data-items.component.ts b/src/app/shared/components/template/components/data-items/data-items.component.ts index 7888ca3063..49bbaea117 100644 --- a/src/app/shared/components/template/components/data-items/data-items.component.ts +++ b/src/app/shared/components/template/components/data-items/data-items.component.ts @@ -66,7 +66,7 @@ export class TmplDataItemsComponent extends TemplateBaseComponent implements OnD this.dataQuery$ = query.pipe(debounceTime(50)).subscribe(async (data) => { // By default, sort the items into the order they appeared in the original data list. // Can be overridden with a `sort` operator applied in the data-items component params - const sortedData = data.sort((a, b) => a.index_original - b.index_original); + const sortedData = data.sort((a, b) => a.row_index - b.row_index); await this.renderItems(sortedData, this._row.rows, this.parameterList); }); } else { diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts index 6b58b792cd..5b5f6a4c28 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts @@ -146,8 +146,8 @@ describe("DynamicDataService", () => { it("adds metadata (original index) to docs", async () => { const obs = await service.query$("data_list", "test_flow"); const data = await firstValueFrom(obs); - expect(data[0].index_original).toEqual(0); - expect(data[1].index_original).toEqual(1); + expect(data[0].row_index).toEqual(0); + expect(data[1].row_index).toEqual(1); }); // QA diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.ts index 50edb79a5e..e2e5a3985e 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.ts @@ -182,7 +182,7 @@ export class DynamicDataService extends AsyncServiceBase { const initialDataWithMeta = initialData.map((el) => { return { ...el, - index_original: initialData.indexOf(el), + row_index: initialData.indexOf(el), }; }); From a16736f7c692e5891f7aaa24c81ccd9d8ff82b6a Mon Sep 17 00:00:00 2001 From: Johnny McQuade Date: Thu, 9 May 2024 11:36:03 +0100 Subject: [PATCH 3/6] chore: include metadata field in memory adapters schemas; return dynamic data queries sorted by row_index by default --- .../data-items/data-items.component.ts | 5 +---- .../dynamic-data/adapters/persistedMemory.ts | 11 ++++++++--- .../dynamic-data/adapters/reactiveMemory.ts | 14 +++++++++++--- .../dynamic-data/dynamic-data.service.spec.ts | 19 ++++++++++++++----- .../dynamic-data/dynamic-data.service.ts | 4 ++++ 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/app/shared/components/template/components/data-items/data-items.component.ts b/src/app/shared/components/template/components/data-items/data-items.component.ts index 315ae7a646..5311a83580 100644 --- a/src/app/shared/components/template/components/data-items/data-items.component.ts +++ b/src/app/shared/components/template/components/data-items/data-items.component.ts @@ -64,10 +64,7 @@ export class TmplDataItemsComponent extends TemplateBaseComponent implements OnD await this.dynamicDataService.ready(); const query = await this.dynamicDataService.query$("data_list", this.dataListName); this.dataQuery$ = query.pipe(debounceTime(50)).subscribe(async (data) => { - // By default, sort the items into the order they appeared in the original data list. - // Can be overridden with a `sort` operator applied in the data-items component params - const sortedData = data.sort((a, b) => a.row_index - b.row_index); - await this.renderItems(sortedData, this._row.rows, this.parameterList); + await this.renderItems(data, this._row.rows, this.parameterList); }); } else { await this.renderItems([], [], {}); diff --git a/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts b/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts index 3923a7f1e6..83a212cbbc 100644 --- a/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts +++ b/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts @@ -37,7 +37,7 @@ const SCHEMA: RxJsonSchema = { title: "base schema for id-primary key data", // NOTE - important to start at 0 and not timestamp (e.g. 20221220) as will check // for migration strategies for each version which is hugely inefficient - version: 1, + version: 2, primaryKey: "id", type: "object", properties: { @@ -48,17 +48,22 @@ const SCHEMA: RxJsonSchema = { flow_name: { type: "string", maxLength: 64 }, flow_type: { type: "string", maxLength: 64 }, row_id: { type: "string", maxLength: 64 }, + row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1 }, data: { type: "object", }, }, - required: ["id", "flow_type", "flow_name", "row_id", "data"], - indexes: ["flow_type", "flow_name", "row_id"], + required: ["id", "flow_type", "flow_name", "row_id", "data", "row_index"], + indexes: ["flow_type", "flow_name", "row_id", "row_index"], }; const MIGRATIONS: MigrationStrategies = { // As part of RXDb v14 update all data requires migrating to change metadata fields (no doc data changes) // https://rxdb.info/releases/14.0.0.html 1: (doc) => doc, + 2: (oldDoc) => { + const newDoc = { ...oldDoc, row_index: 0 }; + return newDoc; + }, }; interface IDataUpdate { diff --git a/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts b/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts index 64fe95c822..4aa4675029 100644 --- a/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts +++ b/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts @@ -2,6 +2,7 @@ import { addRxPlugin, createRxDatabase, MangoQuery, + MigrationStrategies, RxCollection, RxCollectionCreator, RxDatabase, @@ -32,7 +33,7 @@ export const REACTIVE_SCHEMA_BASE: RxJsonSchema = { title: "base schema for id-primary key data", // NOTE - important to start at 0 and not timestamp (e.g. 20221220) as will check // for migration strategies for each version which is hugely inefficient - version: 0, + version: 1, primaryKey: "id", type: "object", properties: { @@ -40,9 +41,16 @@ export const REACTIVE_SCHEMA_BASE: RxJsonSchema = { type: "string", maxLength: 128, // <- the primary key must have set maxLength }, + row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1 }, }, required: ["id"], - indexes: [], + indexes: ["row_index"], +}; +const MIGRATIONS: MigrationStrategies = { + 1: (oldDoc) => { + const newDoc = { ...oldDoc, row_index: 0 }; + return newDoc; + }, }; interface IDataUpdate { @@ -97,7 +105,7 @@ export class ReactiveMemoryAdapater { */ public async createCollection(name: string, schema: RxJsonSchema) { const collections: { [x: string]: RxCollectionCreator } = {}; - collections[name] = { schema }; + collections[name] = { schema, migrationStrategies: MIGRATIONS }; await this.db.addCollections(collections); const collection = this.db.collections[name]; // HACK - sometimes rxdb keeps data in memory during repeated create/delete cycles diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts index 5b5f6a4c28..9bd3fac0b4 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts @@ -9,6 +9,7 @@ import { MockAppDataService } from "../data/app-data.service.spec"; const TEST_DATA_ROWS = [ { id: "id1", number: 1, string: "hello", boolean: true }, { id: "id2", number: 2, string: "goodbye", boolean: false }, + { id: "id0", number: 3, string: "goodbye", boolean: false }, ]; type ITestRow = (typeof TEST_DATA_ROWS)[number]; @@ -59,7 +60,7 @@ describe("DynamicDataService", () => { it("populates initial flows from json", async () => { const obs = await service.query$("data_list", "test_flow"); const data = await firstValueFrom(obs); - expect(data.length).toEqual(2); + expect(data.length).toEqual(3); }); it("supports partial flow row updates", async () => { @@ -85,7 +86,7 @@ describe("DynamicDataService", () => { await service.resetFlow("data_list", "test_flow"); const queryResults: ITestRow[][] = []; const obs = await service.query$("data_list", "test_flow", { - selector: { number: { $gt: 1 } }, + selector: { number: { $gt: 2 } }, }); obs.subscribe((v) => { queryResults.push(v as ITestRow[]); @@ -95,8 +96,8 @@ describe("DynamicDataService", () => { // should have 2 updates, initial result and updated query result expect(queryResults.length).toEqual(2); const [beforeQuery, afterQuery] = queryResults; - expect(beforeQuery.map((row) => row.id)).toEqual(["id2"]); - expect(afterQuery.map((row) => row.id)).toEqual(["id1", "id2"]); + expect(beforeQuery.map((row) => row.id)).toEqual(["id0"]); + expect(afterQuery.map((row) => row.id)).toEqual(["id1", "id0"]); }); it("Supports parallel requests without recreating collections", async () => { @@ -143,13 +144,21 @@ describe("DynamicDataService", () => { expect(data[1].string).toEqual("sets an item correctly for a given _index"); }); - it("adds metadata (original index) to docs", async () => { + it("adds metadata (row_index) to docs", async () => { const obs = await service.query$("data_list", "test_flow"); const data = await firstValueFrom(obs); expect(data[0].row_index).toEqual(0); expect(data[1].row_index).toEqual(1); }); + it("returns data sorted by row_index", async () => { + const obs = await service.query$("data_list", "test_flow"); + const data = await firstValueFrom(obs); + expect(data[0].id).toEqual("id1"); + expect(data[1].id).toEqual("id2"); + expect(data[2].id).toEqual("id0"); + }); + // QA it("prevents query of non-existent data lists", async () => { let errMsg: string; diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.ts index 911234add0..5a219b8887 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.ts @@ -97,6 +97,10 @@ export class DynamicDataService extends AsyncServiceBase { queryObj?: MangoQuery ) { const { collectionName } = await this.ensureCollection(flow_type, flow_name); + + // by default, use `row_index` as query index to return results sorted on this property + const defaultQueryObj = { index: ["row_index", "id"] }; + queryObj = { ...defaultQueryObj, ...queryObj }; // use a live query to return all documents in the collection, converting // from reactive documents to json data instead let query = this.db.query(collectionName, queryObj); From f16d5cded976760514237e8836d01fd6ed4f9efb Mon Sep 17 00:00:00 2001 From: Johnny McQuade Date: Thu, 9 May 2024 12:02:19 +0100 Subject: [PATCH 4/6] chore: make row_index column 'final' --- .../shared/services/dynamic-data/adapters/persistedMemory.ts | 2 +- src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts b/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts index 83a212cbbc..73d78a4c2d 100644 --- a/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts +++ b/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts @@ -48,7 +48,7 @@ const SCHEMA: RxJsonSchema = { flow_name: { type: "string", maxLength: 64 }, flow_type: { type: "string", maxLength: 64 }, row_id: { type: "string", maxLength: 64 }, - row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1 }, + row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1, final: true }, data: { type: "object", }, diff --git a/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts b/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts index 4aa4675029..1d788dd47d 100644 --- a/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts +++ b/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts @@ -41,7 +41,7 @@ export const REACTIVE_SCHEMA_BASE: RxJsonSchema = { type: "string", maxLength: 128, // <- the primary key must have set maxLength }, - row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1 }, + row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1, final: true }, }, required: ["id"], indexes: ["row_index"], From d989d648b99f300e4dda1527db6d3401bc330e42 Mon Sep 17 00:00:00 2001 From: Johnny McQuade Date: Thu, 9 May 2024 12:28:09 +0100 Subject: [PATCH 5/6] test: dynamic data spec test --- .../services/dynamic-data/dynamic-data.service.spec.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts index 9bd3fac0b4..8318723dbd 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts @@ -159,6 +159,12 @@ describe("DynamicDataService", () => { expect(data[2].id).toEqual("id0"); }); + it("does not allow manual updates to row_index property", async () => { + await expectAsync( + service.update("data_list", "test_flow", "id1", { row_index: 5 }) + ).toBeRejectedWithError(); + }); + // QA it("prevents query of non-existent data lists", async () => { let errMsg: string; From 307225709732cd2f7e8824de78a5384444282590 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 13 May 2024 15:03:25 +0100 Subject: [PATCH 6/6] chore: code tidying --- .../shared/services/dynamic-data/dynamic-data.service.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.ts index 5a219b8887..acbc5ef91f 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.ts @@ -209,10 +209,8 @@ export class DynamicDataService extends AsyncServiceBase { /** Retrive json sheet data and merge with any user writes */ private async getInitialData(flow_type: FlowTypes.FlowType, flow_name: string) { const flowData = await this.appDataService.getSheet(flow_type, flow_name); - const writeData = this.writeCache.get(flow_type, flow_name); - const writeDataArray = Object.entries(writeData || {}).map(([id, v]) => ({ ...v, id })) as - | IDocWithID[] - | []; + const writeData = this.writeCache.get(flow_type, flow_name) || {}; + const writeDataArray: IDocWithID[] = Object.entries(writeData).map(([id, v]) => ({ ...v, id })); const mergedData = this.mergeData(flowData?.rows, writeDataArray); return mergedData; }