Bound every action-log read path

packages/typed-storage/__tests__/index.test.ts

@@ -185,6 +185,32 @@ let EVE: User = {
   groups: []
 };
 
+// Schemas shared by the non-unique-index suites: users indexed by level, and the same collection
+// with no index declared (for simulating writes from before the index existed).
+const INDEXED_SCHEMA = {
+  collections: {
+    users: collection<User>()({
+      primaryKey: "name",
+      nonUniqueIndexes: {
+        byLevel: (user: User) => user.level == 0 ? null : user.level
+      }
+    })
+  }
+};
+const PLAIN_SCHEMA = {
+  collections: { users: collection<User>()({ primaryKey: "name" }) }
+};
+const GROUP_SCHEMA = {
+  collections: {
+    users: collection<User>()({
+      primaryKey: "name",
+      nonUniqueIndexes: {
+        byGroup: (user: User) => user.groups
+      }
+    })
+  }
+};
+
 describe("basic collections with string primary key", () => {
   let mockStorage = makeMockStorage();
   let storage = createTypedStorage(mockStorage, {
@@ -215,7 +241,8 @@ describe("basic collections with string primary key", () => {
   testByName(storage.users);
 });
 
-function testByName(index: UniqueIndex<User, string>) {
+// Accepts both a UniqueIndex and a Collection (which has no rebuild()).
+function testByName(index: Pick<UniqueIndex<User, string>, "get" | "list" | "delete">) {
   it("supports get", () => {
     expect(index.get("alice")).toStrictEqual(ALICE);
     expect(index.get("bob")).toStrictEqual(BOB);
@@ -276,7 +303,8 @@ describe("basic collections with integer primary key", () => {
   testByNumber(storage.users);
 });
 
-function testByNumber(index: UniqueIndex<User, number>) {
+// Accepts both a UniqueIndex and a Collection (which has no rebuild()).
+function testByNumber(index: Pick<UniqueIndex<User, number>, "get" | "list" | "delete">) {
   it("supports get", () => {
     expect(index.get(ALICE.uid)).toStrictEqual(ALICE);
     expect(index.get(BOB.uid)).toStrictEqual(BOB);
@@ -509,16 +537,7 @@ describe("unique index by array", () => {
 
 describe("non-unique index by number", () => {
   let mockStorage = makeMockStorage();
-  let storage = createTypedStorage(mockStorage, {
-    collections: {
-      users: collection<User>()({
-        primaryKey: "name",
-        nonUniqueIndexes: {
-          byLevel: (user: User) => user.level == 0 ? null : user.level
-        }
-      })
-    }
-  });
+  let storage = createTypedStorage(mockStorage, INDEXED_SCHEMA);
 
   expect([...storage.users.byLevel.list()]).toStrictEqual([]);
 
@@ -611,3 +630,181 @@ describe("non-unique index by array", () => {
     expect([...index.list()]).toStrictEqual([BOB, DAVE, DAVE]);
   });
 });
+
+describe("non-unique index rebuild", () => {
+  it("backfills an index declared after records were written", () => {
+    let mockStorage = makeMockStorage();
+    let legacy = createTypedStorage(mockStorage, PLAIN_SCHEMA);
+    legacy.users.put(BOB);
+    legacy.users.put(ALICE);
+    legacy.users.put(CAROL);
+    legacy.users.put(EVE);
+
+    let storage = createTypedStorage(mockStorage, INDEXED_SCHEMA);
+
+    // Declared over pre-existing records, the index starts empty -- and resolving one of those
+    // records would throw on the index's remove.
+    expect([...storage.users.byLevel.list()]).toStrictEqual([]);
+    expect(() => storage.users.put({...ALICE, level: 0})).toThrow("inconsistent");
+
+    storage.users.byLevel.rebuild();
+    expect([...storage.users.byLevel.list()]).toStrictEqual([BOB, ALICE, CAROL]);
+    expect([...storage.users.byLevel.get(8)]).toStrictEqual([ALICE, CAROL]);
+
+    // Writes after the rebuild keep the index consistent, including key removal.
+    storage.users.put({...ALICE, level: 0});
+    expect([...storage.users.byLevel.list()]).toStrictEqual([BOB, CAROL]);
+  });
+
+  it("backfills a multi-key index (array index function)", () => {
+    let mockStorage = makeMockStorage();
+    let legacy = createTypedStorage(mockStorage, PLAIN_SCHEMA);
+    legacy.users.put(ALICE);
+    legacy.users.put(BOB);
+    legacy.users.put(DAVE);
+    legacy.users.put(EVE);  // no groups: unindexed
+
+    let storage = createTypedStorage(mockStorage, GROUP_SCHEMA);
+    storage.users.byGroup.rebuild();
+
+    expect([...storage.users.byGroup.get("everyone")]).toStrictEqual([ALICE, BOB, DAVE]);
+    expect([...storage.users.byGroup.get("admin")]).toStrictEqual([ALICE]);
+    expect([...storage.users.byGroup.get("interns")]).toStrictEqual([DAVE]);
+    expect([...storage.users.byGroup.list({dedupe: true})]).toStrictEqual([ALICE, BOB, DAVE]);
+  });
+
+  it("reclaims orphaned child rows and never reuses child-group ids", () => {
+    let mockStorage = makeMockStorage();
+    let storage = createTypedStorage(mockStorage, INDEXED_SCHEMA);
+    storage.users.put(ALICE);  // level 8
+    storage.users.put(BOB);    // level 4
+
+    // A child row whose parent key vanished (e.g. a partially-applied wipe): unreachable from
+    // the parent keys, but still swept by rebuild's raw-range deleteAll.
+    mockStorage.kv.put("users.byLevel.999:zombie", {});
+    let preIds = new Set([...mockStorage.kv.list({prefix: "users.byLevel:"})].map(([, id]) => id));
+    let counterBefore = mockStorage.kv.get<number>("users.byLevel#")!;
+
+    storage.users.byLevel.rebuild();
+
+    expect(mockStorage.kv.get("users.byLevel.999:zombie")).toStrictEqual(undefined);
+    // The unique-id counter survives the wipe, so the rebuilt groups get fresh ids.
+    expect(mockStorage.kv.get<number>("users.byLevel#")).toBeGreaterThanOrEqual(counterBefore);
+    for (let [, id] of mockStorage.kv.list({prefix: "users.byLevel:"})) {
+      expect(preIds.has(id)).toStrictEqual(false);
+    }
+    expect([...storage.users.byLevel.list()]).toStrictEqual([BOB, ALICE]);
+  });
+
+  it("discards stale entries for records changed behind the index's back", () => {
+    let mockStorage = makeMockStorage();
+    let storage = createTypedStorage(mockStorage, INDEXED_SCHEMA);
+    storage.users.put(ALICE);
+    storage.users.put(DAVE);
+
+    // Mutate through a view without the index, leaving it stale: DAVE gone, BOB unindexed.
+    let legacy = createTypedStorage(mockStorage, PLAIN_SCHEMA);
+    legacy.users.delete("dave");
+    legacy.users.put(BOB);
+
+    storage.users.byLevel.rebuild();
+    expect([...storage.users.byLevel.list()]).toStrictEqual([BOB, ALICE]);
+    expect([...storage.users.byLevel.get(1)]).toStrictEqual([]);
+  });
+});
+
+describe("unique index rebuild", () => {
+  const UNIQUE_SCHEMA = {
+    collections: {
+      users: collection<User>()({
+        primaryKey: "name",
+        uniqueIndexes: {
+          byUid: (user: User) => user.uid === 404 ? null : user.uid
+        }
+      })
+    }
+  };
+
+  it("backfills an index declared after records were written", () => {
+    let mockStorage = makeMockStorage();
+    let legacy = createTypedStorage(mockStorage, PLAIN_SCHEMA);
+    legacy.users.put(ALICE);
+    legacy.users.put(BOB);
+    legacy.users.put(EVE);
+
+    let storage = createTypedStorage(mockStorage, UNIQUE_SCHEMA);
+
+    // Declared over pre-existing records, the index starts empty -- and changing one of those
+    // records' keys would throw on the index's remove.
+    expect(storage.users.byUid.get(ALICE.uid)).toStrictEqual(undefined);
+    expect(() => storage.users.put({...ALICE, uid: 46})).toThrow("inconsistent");
+
+    storage.users.byUid.rebuild();
+    expect(storage.users.byUid.get(ALICE.uid)).toStrictEqual(ALICE);
+    expect(storage.users.byUid.get(BOB.uid)).toStrictEqual(BOB);
+    expect(storage.users.byUid.get(EVE.uid)).toStrictEqual(undefined);  // null-keyed, unindexed
+
+    // Writes after the rebuild keep the index consistent, including key changes.
+    storage.users.put({...ALICE, uid: 46});
+    expect(storage.users.byUid.get(ALICE.uid)).toStrictEqual(undefined);
+    expect(storage.users.byUid.get(46)).toStrictEqual({...ALICE, uid: 46});
+  });
+
+  it("throws when two records derive the same key, leaving no partial index", () => {
+    let mockStorage = makeMockStorage();
+    let legacy = createTypedStorage(mockStorage, PLAIN_SCHEMA);
+    legacy.users.put(ALICE);
+    legacy.users.put({...BOB, uid: ALICE.uid});
+
+    let storage = createTypedStorage(mockStorage, UNIQUE_SCHEMA);
+    expect(() => storage.users.byUid.rebuild()).toThrow("conflicts");
+
+    // The rebuild runs in one transaction: the entries added before the conflict rolled back.
+    expect(storage.users.byUid.get(ALICE.uid)).toStrictEqual(undefined);
+    expect([...storage.users.byUid.list()]).toStrictEqual([]);
+  });
+});
+
+describe("non-unique index ranged get", () => {
+  it("ranges and pages within one group by primary key", () => {
+    let storage = createTypedStorage(makeMockStorage(), GROUP_SCHEMA);
+    storage.users.put(BOB);
+    storage.users.put(DAVE);
+    storage.users.put(CAROL);
+    storage.users.put(ALICE);
+
+    let index = storage.users.byGroup;
+    expect([...index.get("everyone", {start: "bob"})]).toStrictEqual([BOB, CAROL, DAVE]);
+    expect([...index.get("everyone", {startAfter: "bob"})]).toStrictEqual([CAROL, DAVE]);
+    expect([...index.get("everyone", {end: "carol"})]).toStrictEqual([ALICE, BOB]);
+    expect([...index.get("everyone", {limit: 2})]).toStrictEqual([ALICE, BOB]);
+    expect([...index.get("everyone", {reverse: true})]).toStrictEqual([DAVE, CAROL, BOB, ALICE]);
+    expect([...index.get("admin", {reverse: true, limit: 1})]).toStrictEqual([CAROL]);
+    expect([...index.get("nobody", {limit: 2})]).toStrictEqual([]);
+  });
+
+  it("pages a numeric-pk group descending with an exclusive end", () => {
+    type Row = {id: number, group: string};
+    let storage = createTypedStorage(makeMockStorage(), {
+      collections: {
+        rows: collection<Row>()({
+          primaryKey: "id",
+          nonUniqueIndexes: {
+            byGroup: (row: Row) => row.group
+          }
+        })
+      }
+    });
+    for (let id = 0; id < 7; id++) {
+      storage.rows.put({id, group: id % 2 === 0 ? "even" : "odd"});
+    }
+
+    // The cursored-history shape: a newest-first page strictly below the cursor.
+    let index = storage.rows.byGroup;
+    expect([...index.get("even", {end: 6, reverse: true, limit: 2})].map(r => r.id))
+        .toStrictEqual([4, 2]);
+    expect([...index.get("even", {reverse: true, limit: 2})].map(r => r.id))
+        .toStrictEqual([6, 4]);
+    expect([...index.get("even", {end: 2})].map(r => r.id)).toStrictEqual([0]);
+  });
+});