Merge pull request #2312 from IDEMSInternational/fix/data-items-order

fix: data items are sorted according to original order by default

src/app/shared/services/dynamic-data/adapters/persistedMemory.ts

@@ -37,7 +37,7 @@ const SCHEMA: RxJsonSchema<any> = {
   title: "base schema for id-primary key data",
   // NOTE - important to start at 0 and not timestamp (e.g. 20221220) as will check
   // for migration strategies for each version which is hugely inefficient
-  version: 1,
+  version: 2,
   primaryKey: "id",
   type: "object",
   properties: {
@@ -48,17 +48,22 @@ const SCHEMA: RxJsonSchema<any> = {
     flow_name: { type: "string", maxLength: 64 },
     flow_type: { type: "string", maxLength: 64 },
     row_id: { type: "string", maxLength: 64 },
+    row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1, final: true },
     data: {
       type: "object",
     },
   },
-  required: ["id", "flow_type", "flow_name", "row_id", "data"],
-  indexes: ["flow_type", "flow_name", "row_id"],
+  required: ["id", "flow_type", "flow_name", "row_id", "data", "row_index"],
+  indexes: ["flow_type", "flow_name", "row_id", "row_index"],
 };
 const MIGRATIONS: MigrationStrategies = {
   // As part of RXDb v14 update all data requires migrating to change metadata fields (no doc data changes)
   // https://rxdb.info/releases/14.0.0.html
   1: (doc) => doc,
+  2: (oldDoc) => {
+    const newDoc = { ...oldDoc, row_index: 0 };
+    return newDoc;
+  },
 };
 
 interface IDataUpdate {

src/app/shared/services/dynamic-data/adapters/reactiveMemory.ts

@@ -2,6 +2,7 @@ import {
   addRxPlugin,
   createRxDatabase,
   MangoQuery,
+  MigrationStrategies,
   RxCollection,
   RxCollectionCreator,
   RxDatabase,
@@ -32,17 +33,24 @@ export const REACTIVE_SCHEMA_BASE: RxJsonSchema<any> = {
   title: "base schema for id-primary key data",
   // NOTE - important to start at 0 and not timestamp (e.g. 20221220) as will check
   // for migration strategies for each version which is hugely inefficient
-  version: 0,
+  version: 1,
   primaryKey: "id",
   type: "object",
   properties: {
     id: {
       type: "string",
       maxLength: 128, // <- the primary key must have set maxLength
     },
+    row_index: { type: "integer", minimum: 0, maximum: 10000, multipleOf: 1, final: true },
   },
   required: ["id"],
-  indexes: [],
+  indexes: ["row_index"],
+};
+const MIGRATIONS: MigrationStrategies = {
+  1: (oldDoc) => {
+    const newDoc = { ...oldDoc, row_index: 0 };
+    return newDoc;
+  },
 };
 
 interface IDataUpdate {
@@ -97,7 +105,7 @@ export class ReactiveMemoryAdapater {
    */
   public async createCollection(name: string, schema: RxJsonSchema<any>) {
     const collections: { [x: string]: RxCollectionCreator<any> } = {};
-    collections[name] = { schema };
+    collections[name] = { schema, migrationStrategies: MIGRATIONS };
     await this.db.addCollections(collections);
     const collection = this.db.collections[name];
     // HACK - sometimes rxdb keeps data in memory during repeated create/delete cycles

src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts

@@ -9,6 +9,7 @@ import { MockAppDataService } from "../data/app-data.service.spec";
 const TEST_DATA_ROWS = [
   { id: "id1", number: 1, string: "hello", boolean: true },
   { id: "id2", number: 2, string: "goodbye", boolean: false },
+  { id: "id0", number: 3, string: "goodbye", boolean: false },
 ];
 type ITestRow = (typeof TEST_DATA_ROWS)[number];
 
@@ -59,7 +60,7 @@ describe("DynamicDataService", () => {
   it("populates initial flows from json", async () => {
     const obs = await service.query$("data_list", "test_flow");
     const data = await firstValueFrom(obs);
-    expect(data.length).toEqual(2);
+    expect(data.length).toEqual(3);
   });
 
   it("supports partial flow row updates", async () => {
@@ -85,7 +86,7 @@ describe("DynamicDataService", () => {
     await service.resetFlow("data_list", "test_flow");
     const queryResults: ITestRow[][] = [];
     const obs = await service.query$("data_list", "test_flow", {
-      selector: { number: { $gt: 1 } },
+      selector: { number: { $gt: 2 } },
     });
     obs.subscribe((v) => {
       queryResults.push(v as ITestRow[]);
@@ -95,8 +96,8 @@ describe("DynamicDataService", () => {
     // should have 2 updates, initial result and updated query result
     expect(queryResults.length).toEqual(2);
     const [beforeQuery, afterQuery] = queryResults;
-    expect(beforeQuery.map((row) => row.id)).toEqual(["id2"]);
-    expect(afterQuery.map((row) => row.id)).toEqual(["id1", "id2"]);
+    expect(beforeQuery.map((row) => row.id)).toEqual(["id0"]);
+    expect(afterQuery.map((row) => row.id)).toEqual(["id1", "id0"]);
   });
 
   it("Supports parallel requests without recreating collections", async () => {
@@ -143,6 +144,27 @@ describe("DynamicDataService", () => {
     expect(data[1].string).toEqual("sets an item correctly for a given _index");
   });
 
+  it("adds metadata (row_index) to docs", async () => {
+    const obs = await service.query$<any>("data_list", "test_flow");
+    const data = await firstValueFrom(obs);
+    expect(data[0].row_index).toEqual(0);
+    expect(data[1].row_index).toEqual(1);
+  });
+
+  it("returns data sorted by row_index", async () => {
+    const obs = await service.query$<any>("data_list", "test_flow");
+    const data = await firstValueFrom(obs);
+    expect(data[0].id).toEqual("id1");
+    expect(data[1].id).toEqual("id2");
+    expect(data[2].id).toEqual("id0");
+  });
+
+  it("does not allow manual updates to row_index property", async () => {
+    await expectAsync(
+      service.update("data_list", "test_flow", "id1", { row_index: 5 })
+    ).toBeRejectedWithError();
+  });
+
   // QA
   it("prevents query of non-existent data lists", async () => {
     let errMsg: string;

src/app/shared/services/dynamic-data/dynamic-data.service.ts

@@ -12,7 +12,7 @@ import { ReactiveMemoryAdapater, REACTIVE_SCHEMA_BASE } from "./adapters/reactiv
 import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry";
 import { TopLevelProperty } from "rxdb/dist/types/types";
 
-type IDocWithID = { id: string };
+type IDocWithID = { [key: string]: any; id: string };
 
 @Injectable({ providedIn: "root" })
 /**
@@ -97,6 +97,10 @@ export class DynamicDataService extends AsyncServiceBase {
     queryObj?: MangoQuery
   ) {
     const { collectionName } = await this.ensureCollection(flow_type, flow_name);
+
+    // by default, use `row_index` as query index to return results sorted on this property
+    const defaultQueryObj = { index: ["row_index", "id"] };
+    queryObj = { ...defaultQueryObj, ...queryObj };
     // use a live query to return all documents in the collection, converting
     // from reactive documents to json data instead
     let query = this.db.query(collectionName, queryObj);
@@ -185,9 +189,17 @@ export class DynamicDataService extends AsyncServiceBase {
     if (initialData.length === 0) {
       throw new Error(`No data exists for collection [${flow_name}], cannot initialise`);
     }
-    const schema = this.inferSchema(initialData[0]);
+    // add index property to each element before insert, for sorting queried data by original order
+    const initialDataWithMeta = initialData.map((el) => {
+      return {
+        ...el,
+        row_index: initialData.indexOf(el),
+      };
+    });
+
+    const schema = this.inferSchema(initialDataWithMeta[0]);
     await this.db.createCollection(collectionName, schema);
-    await this.db.bulkInsert(collectionName, initialData);
+    await this.db.bulkInsert(collectionName, initialDataWithMeta);
     // notify any observers that collection has been created
     this.collectionCreators[collectionName].next(collectionName);
     this.collectionCreators[collectionName].complete();
@@ -197,8 +209,8 @@ export class DynamicDataService extends AsyncServiceBase {
   /** Retrive json sheet data and merge with any user writes */
   private async getInitialData(flow_type: FlowTypes.FlowType, flow_name: string) {
     const flowData = await this.appDataService.getSheet(flow_type, flow_name);
-    const writeData = this.writeCache.get(flow_type, flow_name);
-    const writeDataArray = Object.entries(writeData || {}).map(([id, v]) => ({ ...v, id }));
+    const writeData = this.writeCache.get(flow_type, flow_name) || {};
+    const writeDataArray: IDocWithID[] = Object.entries(writeData).map(([id, v]) => ({ ...v, id }));
     const mergedData = this.mergeData(flowData?.rows, writeDataArray);
     return mergedData;
   }
@@ -208,7 +220,7 @@ export class DynamicDataService extends AsyncServiceBase {
     return `${flow_type}${flow_name}`.toLowerCase().replace(/[^a-z0-9]/g, "");
   }
 
-  private mergeData<T extends { id: string }>(flowData: T[] = [], dbData: T[] = []) {
+  private mergeData<T extends IDocWithID>(flowData: T[] = [], dbData: T[] = []) {
     const flowHashmap = arrayToHashmap(flowData, "id");
     const dbDataHashmap = arrayToHashmap(dbData, "id");
     const merged = deepMergeObjects(flowHashmap, dbDataHashmap);