diff --git a/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts b/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts index 3923a7f1e6..73d78a4c2d 100644 --- a/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts +++ b/src/app/shared/services/dynamic-data/adapters/persistedMemory.ts @@ -37,7 +37,7 @@ const SCHEMA: RxJsonSchema = { title: "base schema for id-primary key data", // NOTE - important to start at 0 and not timestamp (e.g. 20221220) as will check // for migration strategies for each version which is hugely inefficient - version: 1, + version: 2, primaryKey: "id", type: "object", properties: { @@ -48,17 +48,22 @@ const SCHEMA: RxJsonSchema = { flow_name: { type: "string", maxLength: 64 }, flow_type: { type: "string", maxLength: 64 }, row_id: { type: "string", maxLength: 64 }, + row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1, final: true }, data: { type: "object", }, }, - required: ["id", "flow_type", "flow_name", "row_id", "data"], - indexes: ["flow_type", "flow_name", "row_id"], + required: ["id", "flow_type", "flow_name", "row_id", "data", "row_index"], + indexes: ["flow_type", "flow_name", "row_id", "row_index"], }; const MIGRATIONS: MigrationStrategies = { // As part of RXDb v14 update all data requires migrating to change metadata fields (no doc data changes) // https://rxdb.info/releases/14.0.0.html 1: (doc) => doc, + 2: (oldDoc) => { + const newDoc = { ...oldDoc, row_index: 0 }; + return newDoc; + }, }; interface IDataUpdate { diff --git a/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts b/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts index 64fe95c822..1d788dd47d 100644 --- a/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts +++ b/src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts @@ -2,6 +2,7 @@ import { addRxPlugin, createRxDatabase, MangoQuery, + MigrationStrategies, RxCollection, RxCollectionCreator, RxDatabase, @@ -32,7 +33,7 @@ export const REACTIVE_SCHEMA_BASE: RxJsonSchema = { title: "base schema for id-primary key data", // NOTE - important to start at 0 and not timestamp (e.g. 20221220) as will check // for migration strategies for each version which is hugely inefficient - version: 0, + version: 1, primaryKey: "id", type: "object", properties: { @@ -40,9 +41,16 @@ export const REACTIVE_SCHEMA_BASE: RxJsonSchema = { type: "string", maxLength: 128, // <- the primary key must have set maxLength }, + row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1, final: true }, }, required: ["id"], - indexes: [], + indexes: ["row_index"], +}; +const MIGRATIONS: MigrationStrategies = { + 1: (oldDoc) => { + const newDoc = { ...oldDoc, row_index: 0 }; + return newDoc; + }, }; interface IDataUpdate { @@ -97,7 +105,7 @@ export class ReactiveMemoryAdapater { */ public async createCollection(name: string, schema: RxJsonSchema) { const collections: { [x: string]: RxCollectionCreator } = {}; - collections[name] = { schema }; + collections[name] = { schema, migrationStrategies: MIGRATIONS }; await this.db.addCollections(collections); const collection = this.db.collections[name]; // HACK - sometimes rxdb keeps data in memory during repeated create/delete cycles diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts index 50ec015cc2..8318723dbd 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts @@ -9,6 +9,7 @@ import { MockAppDataService } from "../data/app-data.service.spec"; const TEST_DATA_ROWS = [ { id: "id1", number: 1, string: "hello", boolean: true }, { id: "id2", number: 2, string: "goodbye", boolean: false }, + { id: "id0", number: 3, string: "goodbye", boolean: false }, ]; type ITestRow = (typeof TEST_DATA_ROWS)[number]; @@ -59,7 +60,7 @@ describe("DynamicDataService", () => { it("populates initial flows from json", async () => { const obs = await service.query$("data_list", "test_flow"); const data = await firstValueFrom(obs); - expect(data.length).toEqual(2); + expect(data.length).toEqual(3); }); it("supports partial flow row updates", async () => { @@ -85,7 +86,7 @@ describe("DynamicDataService", () => { await service.resetFlow("data_list", "test_flow"); const queryResults: ITestRow[][] = []; const obs = await service.query$("data_list", "test_flow", { - selector: { number: { $gt: 1 } }, + selector: { number: { $gt: 2 } }, }); obs.subscribe((v) => { queryResults.push(v as ITestRow[]); @@ -95,8 +96,8 @@ describe("DynamicDataService", () => { // should have 2 updates, initial result and updated query result expect(queryResults.length).toEqual(2); const [beforeQuery, afterQuery] = queryResults; - expect(beforeQuery.map((row) => row.id)).toEqual(["id2"]); - expect(afterQuery.map((row) => row.id)).toEqual(["id1", "id2"]); + expect(beforeQuery.map((row) => row.id)).toEqual(["id0"]); + expect(afterQuery.map((row) => row.id)).toEqual(["id1", "id0"]); }); it("Supports parallel requests without recreating collections", async () => { @@ -143,6 +144,27 @@ describe("DynamicDataService", () => { expect(data[1].string).toEqual("sets an item correctly for a given _index"); }); + it("adds metadata (row_index) to docs", async () => { + const obs = await service.query$("data_list", "test_flow"); + const data = await firstValueFrom(obs); + expect(data[0].row_index).toEqual(0); + expect(data[1].row_index).toEqual(1); + }); + + it("returns data sorted by row_index", async () => { + const obs = await service.query$("data_list", "test_flow"); + const data = await firstValueFrom(obs); + expect(data[0].id).toEqual("id1"); + expect(data[1].id).toEqual("id2"); + expect(data[2].id).toEqual("id0"); + }); + + it("does not allow manual updates to row_index property", async () => { + await expectAsync( + service.update("data_list", "test_flow", "id1", { row_index: 5 }) + ).toBeRejectedWithError(); + }); + // QA it("prevents query of non-existent data lists", async () => { let errMsg: string; diff --git a/src/app/shared/services/dynamic-data/dynamic-data.service.ts b/src/app/shared/services/dynamic-data/dynamic-data.service.ts index 43885a2450..acbc5ef91f 100644 --- a/src/app/shared/services/dynamic-data/dynamic-data.service.ts +++ b/src/app/shared/services/dynamic-data/dynamic-data.service.ts @@ -12,7 +12,7 @@ import { ReactiveMemoryAdapater, REACTIVE_SCHEMA_BASE } from "./adapters/reactiv import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry"; import { TopLevelProperty } from "rxdb/dist/types/types"; -type IDocWithID = { id: string }; +type IDocWithID = { [key: string]: any; id: string }; @Injectable({ providedIn: "root" }) /** @@ -97,6 +97,10 @@ export class DynamicDataService extends AsyncServiceBase { queryObj?: MangoQuery ) { const { collectionName } = await this.ensureCollection(flow_type, flow_name); + + // by default, use `row_index` as query index to return results sorted on this property + const defaultQueryObj = { index: ["row_index", "id"] }; + queryObj = { ...defaultQueryObj, ...queryObj }; // use a live query to return all documents in the collection, converting // from reactive documents to json data instead let query = this.db.query(collectionName, queryObj); @@ -185,9 +189,17 @@ export class DynamicDataService extends AsyncServiceBase { if (initialData.length === 0) { throw new Error(`No data exists for collection [${flow_name}], cannot initialise`); } - const schema = this.inferSchema(initialData[0]); + // add index property to each element before insert, for sorting queried data by original order + const initialDataWithMeta = initialData.map((el) => { + return { + ...el, + row_index: initialData.indexOf(el), + }; + }); + + const schema = this.inferSchema(initialDataWithMeta[0]); await this.db.createCollection(collectionName, schema); - await this.db.bulkInsert(collectionName, initialData); + await this.db.bulkInsert(collectionName, initialDataWithMeta); // notify any observers that collection has been created this.collectionCreators[collectionName].next(collectionName); this.collectionCreators[collectionName].complete(); @@ -197,8 +209,8 @@ export class DynamicDataService extends AsyncServiceBase { /** Retrive json sheet data and merge with any user writes */ private async getInitialData(flow_type: FlowTypes.FlowType, flow_name: string) { const flowData = await this.appDataService.getSheet(flow_type, flow_name); - const writeData = this.writeCache.get(flow_type, flow_name); - const writeDataArray = Object.entries(writeData || {}).map(([id, v]) => ({ ...v, id })); + const writeData = this.writeCache.get(flow_type, flow_name) || {}; + const writeDataArray: IDocWithID[] = Object.entries(writeData).map(([id, v]) => ({ ...v, id })); const mergedData = this.mergeData(flowData?.rows, writeDataArray); return mergedData; } @@ -208,7 +220,7 @@ export class DynamicDataService extends AsyncServiceBase { return `${flow_type}${flow_name}`.toLowerCase().replace(/[^a-z0-9]/g, ""); } - private mergeData(flowData: T[] = [], dbData: T[] = []) { + private mergeData(flowData: T[] = [], dbData: T[] = []) { const flowHashmap = arrayToHashmap(flowData, "id"); const dbDataHashmap = arrayToHashmap(dbData, "id"); const merged = deepMergeObjects(flowHashmap, dbDataHashmap);