Reduce memory usage during index construction. (#8834)

Author: isoos

app/bin/tools/isolate_search_benchmark.dart

@@ -2,6 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:io';
 import 'dart:math';
 
 import 'package:_pub_shared/search/search_form.dart';
@@ -23,13 +24,15 @@ final queries = [
 ];
 
 Future<void> main(List<String> args) async {
-  print('Loading...');
+  print('Started. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB,  '
+      'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
   final primaryRunner = await startSearchIsolate(snapshot: args.first);
   final reducedRunner = await startSearchIsolate(
     snapshot: args.first,
     removeTextContent: true,
   );
-  print('Loaded.');
+  print('Loaded. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB,  '
+      'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
 
   for (var i = 0; i < 5; i++) {
     await _benchmark(primaryRunner, primaryRunner);
@@ -39,6 +42,8 @@ Future<void> main(List<String> args) async {
 
   await primaryRunner.close();
   await reducedRunner.close();
+  print('Done. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB,  '
+      'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
 }
 
 Future<void> _benchmark(IsolateRunner primary, IsolateRunner reduced) async {

app/bin/tools/sdk_search_benchmark.dart

@@ -2,11 +2,17 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:io';
+
 import 'package:pub_dev/search/sdk_mem_index.dart';
 
 /// Loads a Dart SDK search snapshot and executes queries on it, benchmarking their total time to complete.
 Future<void> main() async {
+  print('Started. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB,  '
+      'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
   final index = await createSdkMemIndex();
+  print('Loaded. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB,  '
+      'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
 
   // NOTE: please add more queries to this list, especially if there is a performance bottleneck.
   final queries = [
@@ -25,4 +31,6 @@ Future<void> main() async {
   }
   sw.stop();
   print('${(sw.elapsedMilliseconds / count).toStringAsFixed(2)} ms/request');
+  print('Done. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB,  '
+      'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
 }

app/bin/tools/search_benchmark.dart

@@ -2,14 +2,21 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:async';
+import 'dart:io';
+
 import 'package:_pub_shared/search/search_form.dart';
 import 'package:pub_dev/search/search_service.dart';
 import 'package:pub_dev/search/updater.dart';
 
 /// Loads a search snapshot and executes queries on it, benchmarking their total time to complete.
 Future<void> main(List<String> args) async {
+  print('Started. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB,  '
+      'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
   // Assumes that the first argument is a search snapshot file.
   final index = await loadInMemoryPackageIndexFromFile(args.first);
+  print('Loaded. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB,  '
+      'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
 
   // NOTE: please add more queries to this list, especially if there is a performance bottleneck.
   final queries = [
@@ -34,4 +41,6 @@ Future<void> main(List<String> args) async {
   }
   sw.stop();
   print('${(sw.elapsedMilliseconds / count).toStringAsFixed(2)} ms/request');
+  print('Done. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB,  '
+      'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
 }

app/lib/search/mem_index.dart

@@ -10,7 +10,6 @@ import 'package:collection/collection.dart';
 import 'package:logging/logging.dart';
 import 'package:meta/meta.dart';
 import 'package:pub_dev/search/heap.dart';
-import 'package:pub_dev/service/topics/models.dart';
 import 'package:pub_dev/third_party/bit_array/bit_array.dart';
 
 import 'models.dart';
@@ -23,7 +22,6 @@ final _textSearchTimeout = Duration(milliseconds: 500);
 
 class InMemoryPackageIndex {
   final List<PackageDocument> _documents;
-  final _documentsByName = <String, PackageDocument>{};
   final _nameToIndex = <String, int>{};
   late final PackageNameIndex _packageNameIndex;
   late final TokenIndex<String> _descrIndex;
@@ -47,23 +45,15 @@ class InMemoryPackageIndex {
   late final List<IndexedPackageHit> _pointsOrderedHits;
   late final List<IndexedPackageHit> _trendingOrderedHits;
 
-  // Contains all of the topics the index had seen so far.
-  // TODO: consider moving this into a separate index
-  // TODO: get the list of topics from the bucket
-  final _topics = <String>{
-    ...canonicalTopics.aliasToCanonicalMap.values,
-  };
-
   late final DateTime _lastUpdated;
 
   InMemoryPackageIndex({
     required Iterable<PackageDocument> documents,
   }) : _documents = [...documents] {
     final apiDocPageKeys = <IndexedApiDocPage>[];
-    final apiDocPageValues = <String>[];
+    final apiDocPageValues = <List<String>>[];
     for (var i = 0; i < _documents.length; i++) {
       final doc = _documents[i];
-      _documentsByName[doc.package] = doc;
       _nameToIndex[doc.package] = i;
 
       // transform tags into numberical IDs
@@ -78,16 +68,10 @@ class InMemoryPackageIndex {
         for (final page in apiDocPages) {
           if (page.symbols != null && page.symbols!.isNotEmpty) {
             apiDocPageKeys.add(IndexedApiDocPage(i, page));
-            apiDocPageValues.add(page.symbols!.join(' '));
+            apiDocPageValues.add(page.symbols!);
           }
         }
       }
-
-      // Note: we are not removing topics from this set, only adding them, no
-      //       need for tracking the current topic count.
-      _topics.addAll(doc.tags
-          .where((t) => t.startsWith('topic:'))
-          .map((t) => t.split('topic:').last));
     }
 
     final packageKeys = _documents.map((d) => d.package).toList();
@@ -101,7 +85,7 @@ class InMemoryPackageIndex {
       packageKeys,
       _documents.map((d) => d.readme).toList(),
     );
-    _apiSymbolIndex = TokenIndex(apiDocPageKeys, apiDocPageValues);
+    _apiSymbolIndex = TokenIndex.fromValues(apiDocPageKeys, apiDocPageValues);
 
     // update download scores only if they were not set (should happen on old runtime's snapshot and local tests)
     if (_documents.any((e) => e.downloadScore == null)) {
@@ -131,7 +115,7 @@ class InMemoryPackageIndex {
   IndexInfo indexInfo() {
     return IndexInfo(
       isReady: true,
-      packageCount: _documentsByName.length,
+      packageCount: _documents.length,
       lastUpdated: _lastUpdated,
     );
   }
@@ -363,7 +347,7 @@ class InMemoryPackageIndex {
     }
 
     // exact package name
-    if (_documentsByName.containsKey(parsedQueryText)) {
+    if (_nameToIndex.containsKey(parsedQueryText)) {
       return parsedQueryText;
     }
 
@@ -377,12 +361,9 @@ class InMemoryPackageIndex {
     }
     // Note: to keep it simple, we select the most downloaded one from competing matches.
     return matches.reduce((a, b) {
-      if (_documentsByName[a]!.downloadCount >
-          _documentsByName[b]!.downloadCount) {
-        return a;
-      } else {
-        return b;
-      }
+      final aDoc = _documents[_nameToIndex[a]!];
+      final bDoc = _documents[_nameToIndex[b]!];
+      return aDoc.downloadCount > bDoc.downloadCount ? a : b;
     });
   }
 
@@ -487,7 +468,7 @@ class InMemoryPackageIndex {
             packageScores.setValueMaxOf(doc.index, value);
 
             // add the page and re-sort the current results
-            pages.add(MapEntry(doc.page.relativePath, value));
+            pages.add(MapEntry(doc.relativePath, value));
             if (pages.length > 1) {
               pages.sort((a, b) => -a.value.compareTo(b.value));
             }
@@ -750,7 +731,8 @@ class IndexedPackageHit {
 
 class IndexedApiDocPage {
   final int index;
-  final ApiDocPage page;
+  final String relativePath;
 
-  IndexedApiDocPage(this.index, this.page);
+  IndexedApiDocPage(this.index, ApiDocPage page)
+      : relativePath = page.relativePath;
 }

app/lib/search/token_index.dart

@@ -51,21 +51,43 @@ class TokenIndex<K> {
       if (text == null) {
         continue;
       }
-      final tokens = tokenize(text);
-      if (tokens == null || tokens.isEmpty) {
+      _build(i, text, skipDocumentWeight);
+    }
+  }
+
+  TokenIndex.fromValues(
+    List<K> ids,
+    List<List<String>?> values, {
+    bool skipDocumentWeight = false,
+  }) : _ids = ids {
+    assert(ids.length == values.length);
+    final length = values.length;
+    for (var i = 0; i < length; i++) {
+      final parts = values[i];
+
+      if (parts == null || parts.isEmpty) {
         continue;
       }
-      // Document weight is a highly scaled-down proxy of the length.
-      final dw =
-          skipDocumentWeight ? 1.0 : 1 + math.log(1 + tokens.length) / 100;
-      for (final e in tokens.entries) {
-        final token = e.key;
-        final weights = _inverseIds.putIfAbsent(token, () => {});
-        weights[i] = math.max(weights[i] ?? 0.0, e.value / dw);
+      for (final text in parts) {
+        _build(i, text, skipDocumentWeight);
       }
     }
   }
 
+  void _build(int i, String text, bool skipDocumentWeight) {
+    final tokens = tokenize(text);
+    if (tokens == null || tokens.isEmpty) {
+      return;
+    }
+    // Document weight is a highly scaled-down proxy of the length.
+    final dw = skipDocumentWeight ? 1.0 : 1 + math.log(1 + tokens.length) / 100;
+    for (final e in tokens.entries) {
+      final token = e.key;
+      final weights = _inverseIds.putIfAbsent(token, () => {});
+      weights[i] = math.max(weights[i] ?? 0.0, e.value / dw);
+    }
+  }
+
   factory TokenIndex.fromMap(Map<K, String> map) {
     final keys = map.keys.toList();
     final values = map.values.toList();