diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/BUILD index be993a96d6..651b0ba2e0 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/BUILD +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/BUILD @@ -35,7 +35,9 @@ java_library( deps = [ ":search_service_logic", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:dimension_overlay", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:overlay_view", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:device_corpus", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:fleet_chip_resolver", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:fleet_column_cataloger", @@ -48,6 +50,7 @@ java_library( "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:host_corpus", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:scenario_curation", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:search_corpus", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:dimension_overlay_store", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:fleet_snapshot_store", "@maven//:com_google_guava_guava", "@maven//:javax_inject_jsr330_api", diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImpl.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImpl.java index 8b6c33fa04..01c56597b6 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImpl.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImpl.java @@ -18,9 +18,11 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.devtools.mobileharness.fe.v6.service.proto.search.Filter; import com.google.devtools.mobileharness.fe.v6.service.proto.search.Fleet; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetChipResolverRequest; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetChipResolverResponse; @@ -42,7 +44,9 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetValueListRequest; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetValueListResponse; import com.google.devtools.mobileharness.fe.v6.service.proto.search.SearchEntity; +import com.google.devtools.mobileharness.fe.v6.service.search.index.DimensionOverlay; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; +import com.google.devtools.mobileharness.fe.v6.service.search.index.OverlayView; import com.google.devtools.mobileharness.fe.v6.service.search.query.DeviceCorpus; import com.google.devtools.mobileharness.fe.v6.service.search.query.FleetChipResolver; import com.google.devtools.mobileharness.fe.v6.service.search.query.FleetColumnCataloger; @@ -55,8 +59,11 @@ import com.google.devtools.mobileharness.fe.v6.service.search.query.HostCorpus; import com.google.devtools.mobileharness.fe.v6.service.search.query.ScenarioCuration; import com.google.devtools.mobileharness.fe.v6.service.search.query.SearchCorpus; +import com.google.devtools.mobileharness.fe.v6.service.search.refresh.DimensionOverlayStore; import com.google.devtools.mobileharness.fe.v6.service.search.refresh.FleetSnapshotStore; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import javax.inject.Inject; import javax.inject.Singleton; @@ -82,6 +89,7 @@ public final class SearchServiceLogicImpl implements SearchServiceLogic { private final ListeningExecutorService executor; private final FleetSnapshotStore store; + private final DimensionOverlayStore overlayStore; private final Map curations; private final FleetSearchConfigProvider searchConfigProvider; private final FleetFlatSearcher flatSearcher; @@ -96,6 +104,7 @@ public final class SearchServiceLogicImpl implements SearchServiceLogic { SearchServiceLogicImpl( ListeningExecutorService executor, FleetSnapshotStore store, + DimensionOverlayStore overlayStore, Map curations, FleetSearchConfigProvider searchConfigProvider, FleetFlatSearcher flatSearcher, @@ -107,6 +116,7 @@ public final class SearchServiceLogicImpl implements SearchServiceLogic { FleetColumnCataloger columnCataloger) { this.executor = executor; this.store = store; + this.overlayStore = overlayStore; this.curations = curations; this.searchConfigProvider = searchConfigProvider; this.flatSearcher = flatSearcher; @@ -134,12 +144,18 @@ public ListenableFuture getFleetSearchConfig( @Override public ListenableFuture searchFleet(FleetSearchRequest request) { - return Futures.submit(() -> searchFleetSync(request), executor); + Fleet fleet = normalize(request.getFleet()); + Set referencedKeys = extractReferencedKeys(request); + return Futures.transformAsync( + overlayStore.loadOverlaysAsync(fleet, referencedKeys, executor), + overlays -> Futures.submit(() -> searchFleetSync(request, overlays), executor), + executor); } - private FleetSearchResults searchFleetSync(FleetSearchRequest request) { + private FleetSearchResults searchFleetSync( + FleetSearchRequest request, ImmutableMap overlays) { Fleet fleet = normalize(request.getFleet()); - SearchCorpus corpus = corpus(fleet, request.getEntity()); + SearchCorpus corpus = corpus(fleet, request.getEntity(), overlays); return switch (request.getViewCase()) { case FLAT -> { FleetFlatView flat = request.getFlat(); @@ -174,8 +190,6 @@ private FleetSearchResults searchFleetSync(FleetSearchRequest request) { expand.getPageToken()); yield FleetSearchResults.newBuilder().setFlat(results).build(); } - // A request with no view selects no results shape, so return an empty result rather than - // guessing a view. case VIEW_NOT_SET -> FleetSearchResults.getDefaultInstance(); }; } @@ -196,9 +210,6 @@ public ListenableFuture resolveFleetChips( FleetChipResolverRequest request) { return Futures.submit( () -> { - // Chip resolution is stateless: the request carries no fleet, and the resolver reads only - // key display names and value casing, which are the same across fleets. Read the self - // snapshot. FleetSnapshot snapshot = store.get(Fleet.FLEET_SELF); return chipResolver.resolve(snapshot, request); }, @@ -207,12 +218,18 @@ public ListenableFuture resolveFleetChips( @Override public ListenableFuture getFleetValueList(FleetValueListRequest request) { - return Futures.submit( - () -> { - Fleet fleet = normalize(request.getFleet()); - return valueLister.listValues( - corpus(fleet, request.getEntity()), request.getKey(), request.getFiltersList()); - }, + Fleet fleet = normalize(request.getFleet()); + Set keys = extractReferencedKeys(request); + return Futures.transformAsync( + overlayStore.loadOverlaysAsync(fleet, keys, executor), + overlays -> + Futures.submit( + () -> + valueLister.listValues( + corpus(fleet, request.getEntity(), overlays), + request.getKey(), + request.getFiltersList()), + executor), executor); } @@ -238,22 +255,80 @@ public ListenableFuture getFleetColumnCatalog( executor); } - /** - * Builds the search corpus for a fleet and entity. A host search projects the fleet through a - * {@link HostCorpus} over the host index and host posting lists; every other entity projects it - * through a {@link DeviceCorpus}. A missing curation is passed through as null so the promoted - * keys provider keeps its curation-missing fallback. - */ - private SearchCorpus corpus(Fleet fleet, SearchEntity entity) { + private SearchCorpus corpus( + Fleet fleet, SearchEntity entity, ImmutableMap overlays) { if (entity == SearchEntity.SEARCH_ENTITY_HOST) { return new HostCorpus(store.get(fleet), store.hostPostings(fleet), curations.get(fleet)); } - return new DeviceCorpus(store.get(fleet), store.postings(fleet), curations.get(fleet)); + FleetSnapshot snapshot = store.get(fleet); + OverlayView overlayView = OverlayView.bind(snapshot, overlays); + return new DeviceCorpus(snapshot, store.postings(fleet), curations.get(fleet), overlayView); + } + + private SearchCorpus corpus(Fleet fleet, SearchEntity entity) { + return corpus(fleet, entity, ImmutableMap.of()); + } + + private static Set extractReferencedKeys(FleetSearchRequest request) { + Set keys = new HashSet<>(); + for (Filter filter : request.getFiltersList()) { + if (isOverlayKey(filter.getKey())) { + keys.add(filter.getKey()); + } + } + switch (request.getViewCase()) { + case FLAT -> { + FleetFlatView flat = request.getFlat(); + for (String col : flat.getColumnsList()) { + if (isOverlayKey(col)) { + keys.add(col); + } + } + if (flat.hasSort() && isOverlayKey(flat.getSort().getKey())) { + keys.add(flat.getSort().getKey()); + } + } + case GROUP_HEADER -> { + FleetGroupHeaderView header = request.getGroupHeader(); + for (String gb : header.getGroupByList()) { + if (isOverlayKey(gb)) { + keys.add(gb); + } + } + if (header.hasSort() && isOverlayKey(header.getSort().getField().getGroupKey())) { + keys.add(header.getSort().getField().getGroupKey()); + } + } + case GROUP_EXPAND -> { + FleetGroupExpandView expand = request.getGroupExpand(); + for (String col : expand.getColumnsList()) { + if (isOverlayKey(col)) { + keys.add(col); + } + } + } + case VIEW_NOT_SET -> {} + } + return keys; + } + + private static Set extractReferencedKeys(FleetValueListRequest request) { + Set keys = new HashSet<>(); + if (isOverlayKey(request.getKey())) { + keys.add(request.getKey()); + } + for (Filter filter : request.getFiltersList()) { + if (isOverlayKey(filter.getKey())) { + keys.add(filter.getKey()); + } + } + return keys; + } + + private static boolean isOverlayKey(String keyId) { + return keyId != null && keyId.startsWith("dim::"); } - /** - * Normalizes an unspecified fleet to the self fleet, matching the {@code Fleet} proto default. - */ private static Fleet normalize(Fleet fleet) { return fleet == Fleet.FLEET_UNSPECIFIED ? Fleet.FLEET_SELF : fleet; } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD index f570a5cb90..9a84ff32be 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD @@ -183,10 +183,25 @@ java_library( ], ) +java_library( + name = "dimension_overlay", + srcs = ["DimensionOverlay.java"], + deps = [ + ":fleet_snapshot", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:dimension_overlay_raw", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) + java_library( name = "overlay_view", srcs = ["OverlayView.java"], - deps = ["@maven//:com_google_guava_guava"], + deps = [ + ":dimension_overlay", + ":fleet_snapshot", + "@maven//:com_google_guava_guava", + ], ) java_library( diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlay.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlay.java new file mode 100644 index 0000000000..f4a6ae38cf --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlay.java @@ -0,0 +1,216 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.primitives.Ints; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.DimensionOverlayRaw; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * In-memory index and posting cache for one on-demand long-tail dimension. + * + *

Holds the snapshot-independent ground truth ({@link #uuidToValues}, {@link #valueCounts}, + * {@link #sortedValues}, {@link #valueDisplays}) derived purely from the pulled dimension data. + * + *

Posting lists ({@code value -> int[] deviceIndex}) are bound to a specific snapshot's device + * ordering via {@link #bind(FleetSnapshot)}. When a snapshot rotates, postings are re-indexed in + * <1 ms against {@link FleetSnapshot#uuidToIndex()}. + */ +public final class DimensionOverlay { + + private final String keyId; + private final ImmutableMap> uuidToValues; + private final ImmutableMap valueCounts; + private final ImmutableList sortedValues; + private final ImmutableMap valueDisplays; + + private volatile SnapshotBoundPostings boundPostings; + + private DimensionOverlay( + String keyId, + ImmutableMap> uuidToValues, + ImmutableMap valueCounts, + ImmutableList sortedValues, + ImmutableMap valueDisplays, + SnapshotBoundPostings boundPostings) { + this.keyId = checkNotNull(keyId); + this.uuidToValues = checkNotNull(uuidToValues); + this.valueCounts = checkNotNull(valueCounts); + this.sortedValues = checkNotNull(sortedValues); + this.valueDisplays = checkNotNull(valueDisplays); + this.boundPostings = checkNotNull(boundPostings); + } + + public String keyId() { + return keyId; + } + + public ImmutableMap> uuidToValues() { + return uuidToValues; + } + + public ImmutableMap valueCounts() { + return valueCounts; + } + + public ImmutableList sortedValues() { + return sortedValues; + } + + public ImmutableMap valueDisplays() { + return valueDisplays; + } + + public int valueCount(String value) { + return valueCounts.getOrDefault(value, 0); + } + + public ImmutableList valuesForUuid(String uuid) { + ImmutableList values = uuidToValues.get(uuid); + return values != null ? values : ImmutableList.of(); + } + + public ImmutableSet lowerValuesForUuid(String uuid) { + ImmutableList values = uuidToValues.get(uuid); + if (values == null || values.isEmpty()) { + return ImmutableSet.of(); + } + return values.stream().map(Ascii::toLowerCase).collect(toImmutableSet()); + } + + /** + * Returns posting lists aligned with the given snapshot. Re-indexes in <1 ms if the snapshot + * has rotated since the last bind. + */ + @CanIgnoreReturnValue + public SnapshotBoundPostings bind(FleetSnapshot snapshot) { + SnapshotBoundPostings current = boundPostings; + if (current.buildTime().equals(snapshot.buildTime())) { + return current; + } + SnapshotBoundPostings updated = reindex(snapshot); + this.boundPostings = updated; + return updated; + } + + private SnapshotBoundPostings reindex(FleetSnapshot snapshot) { + ImmutableMap uuidToIndex = snapshot.uuidToIndex(); + Map> postingsMap = new HashMap<>(); + + for (Map.Entry> entry : uuidToValues.entrySet()) { + String uuid = entry.getKey(); + Integer deviceIndex = uuidToIndex.get(uuid); + if (deviceIndex == null) { + continue; + } + Set seenValues = new HashSet<>(); + for (String rawVal : entry.getValue()) { + String valLower = Ascii.toLowerCase(rawVal); + if (!valLower.isEmpty() && seenValues.add(valLower)) { + postingsMap.computeIfAbsent(valLower, k -> new ArrayList<>()).add(deviceIndex); + } + } + } + + ImmutableMap.Builder frozen = ImmutableMap.builder(); + for (Map.Entry> entry : postingsMap.entrySet()) { + List list = entry.getValue(); + Collections.sort(list); + frozen.put(entry.getKey(), Ints.toArray(list)); + } + return new SnapshotBoundPostings(snapshot.buildTime(), frozen.buildOrThrow()); + } + + /** Builds a {@link DimensionOverlay} from raw pulled data and initial snapshot. */ + public static DimensionOverlay create(DimensionOverlayRaw raw, FleetSnapshot snapshot) { + String keyId = raw.keyId(); + ImmutableMap> uuidToValues = raw.uuidToValues(); + + Map valueCounts = new HashMap<>(); + Map valueDisplays = new HashMap<>(); + Set distinctValues = new HashSet<>(); + + for (ImmutableList values : uuidToValues.values()) { + Set seenForDevice = new HashSet<>(); + for (String rawVal : values) { + String valLower = Ascii.toLowerCase(rawVal); + if (!valLower.isEmpty() && seenForDevice.add(valLower)) { + distinctValues.add(valLower); + valueCounts.merge(valLower, 1, Integer::sum); + valueDisplays.putIfAbsent(valLower, rawVal); + } + } + } + + List sortedList = new ArrayList<>(distinctValues); + Collections.sort(sortedList); + + DimensionOverlay overlay = + new DimensionOverlay( + keyId, + uuidToValues, + ImmutableMap.copyOf(valueCounts), + ImmutableList.copyOf(sortedList), + ImmutableMap.copyOf(valueDisplays), + new SnapshotBoundPostings(Instant.EPOCH, ImmutableMap.of())); + + // Bind to the given snapshot immediately. + overlay.bind(snapshot); + return overlay; + } + + /** Posting lists bound to a specific snapshot build time. */ + public static final class SnapshotBoundPostings { + private final Instant buildTime; + private final ImmutableMap postings; + + SnapshotBoundPostings(Instant buildTime, ImmutableMap postings) { + this.buildTime = buildTime; + this.postings = postings; + } + + public Instant buildTime() { + return buildTime; + } + + public int[] get(String valueLower) { + int[] posting = postings.get(valueLower); + return posting != null ? posting : EMPTY_POSTING; + } + + public ImmutableMap postings() { + return postings; + } + + private static final int[] EMPTY_POSTING = new int[0]; + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilder.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilder.java index fbd550e613..604b15d8b3 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilder.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilder.java @@ -262,9 +262,17 @@ record HostDevices(HostRecord host, List devices) {} // Phase 2: flatten + merge T accumulators (not 43K). List allDevices = new ArrayList<>(); ImmutableList.Builder allHosts = ImmutableList.builder(); + ImmutableMap.Builder uuidToIndex = ImmutableMap.builder(); + int devIdx = 0; for (HostDevices hd : hostDevices) { allHosts.add(hd.host()); - allDevices.addAll(hd.devices()); + for (DeviceRecord device : hd.devices()) { + allDevices.add(device); + if (!device.deviceId().isEmpty()) { + uuidToIndex.put(device.deviceId(), devIdx); + } + devIdx++; + } } Accumulator merged = new Accumulator(); @@ -283,6 +291,7 @@ record HostDevices(HostRecord host, List devices) {} .setHosts(allHosts.build()) .setIndex(merged.toIndex()) .setHostIndex(hostMerged.toIndex()) + .setUuidToIndex(uuidToIndex.buildKeepingLast()) .build(); } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSnapshot.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSnapshot.java index 0d53b4f34d..fc75d24da6 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSnapshot.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSnapshot.java @@ -18,6 +18,7 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.time.Instant; /** @@ -50,6 +51,9 @@ public abstract class FleetSnapshot { /** Inverted index and value index over {@link #hosts()}. */ public abstract FleetIndex hostIndex(); + /** Map from device UUID to integer index in {@link #devices()}, used for overlay alignment. */ + public abstract ImmutableMap uuidToIndex(); + /** Convenience: number of devices in this snapshot. */ public int deviceCount() { return devices().size(); @@ -62,7 +66,7 @@ public int hostCount() { /** Creates a new builder. */ public static Builder builder() { - return new AutoValue_FleetSnapshot.Builder(); + return new AutoValue_FleetSnapshot.Builder().setUuidToIndex(ImmutableMap.of()); } /** An empty snapshot, used as the initial state before the first refresh completes. */ @@ -73,6 +77,7 @@ public static FleetSnapshot empty() { .setHosts(ImmutableList.of()) .setIndex(CoreFleetIndex.empty()) .setHostIndex(CoreFleetIndex.empty()) + .setUuidToIndex(ImmutableMap.of()) .build(); } @@ -89,6 +94,8 @@ public abstract static class Builder { public abstract Builder setHostIndex(FleetIndex hostIndex); + public abstract Builder setUuidToIndex(ImmutableMap uuidToIndex); + public abstract FleetSnapshot build(); } } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/OverlayView.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/OverlayView.java index ff7fa1b8c0..e62b9bc41b 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/OverlayView.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/OverlayView.java @@ -16,6 +16,8 @@ package com.google.devtools.mobileharness.fe.v6.service.search.index; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -47,10 +49,10 @@ public interface OverlayView { /** All posting lists for an overlay key, or empty map if absent. */ ImmutableMap postingsForKey(String keyId); - /** Values for a specific device index and overlay key. */ + /** Lowercased values for a specific device index and overlay key. */ ImmutableSet valuesForKey(int deviceIndex, String keyId); - /** Display values for a specific device index and overlay key. */ + /** Original display values for a specific device index and overlay key. */ ImmutableList displayValues(int deviceIndex, String keyId); /** Empty overlay view, used when no overlays are loaded. */ @@ -58,6 +60,93 @@ static OverlayView empty() { return EmptyOverlayView.INSTANCE; } + /** Creates an overlay view binding the given overlays to the specified snapshot. */ + static OverlayView bind(FleetSnapshot snapshot, ImmutableMap overlays) { + if (overlays.isEmpty()) { + return empty(); + } + return new BoundOverlayView(snapshot, overlays); + } + + /** Bound implementation of {@link OverlayView} over a specific snapshot. */ + final class BoundOverlayView implements OverlayView { + private final FleetSnapshot snapshot; + private final ImmutableMap overlays; + + BoundOverlayView(FleetSnapshot snapshot, ImmutableMap overlays) { + this.snapshot = checkNotNull(snapshot); + this.overlays = checkNotNull(overlays); + } + + @Override + public boolean containsKey(String keyId) { + return overlays.containsKey(keyId); + } + + @Override + public ImmutableSet loadedKeys() { + return overlays.keySet(); + } + + @Override + public ImmutableList sortedValues(String keyId) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.sortedValues() : ImmutableList.of(); + } + + @Override + public ImmutableMap valueCounts(String keyId) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.valueCounts() : ImmutableMap.of(); + } + + @Override + public ImmutableMap valueDisplays(String keyId) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.valueDisplays() : ImmutableMap.of(); + } + + @Override + public int valueCount(String keyId, String value) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.valueCount(value) : 0; + } + + @Override + public int[] getPostings(String keyId, String value) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.bind(snapshot).get(value) : EMPTY_INT_ARRAY; + } + + @Override + public ImmutableMap postingsForKey(String keyId) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.bind(snapshot).postings() : ImmutableMap.of(); + } + + @Override + public ImmutableSet valuesForKey(int deviceIndex, String keyId) { + if (deviceIndex < 0 || deviceIndex >= snapshot.devices().size()) { + return ImmutableSet.of(); + } + String uuid = snapshot.devices().get(deviceIndex).deviceId(); + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.lowerValuesForUuid(uuid) : ImmutableSet.of(); + } + + @Override + public ImmutableList displayValues(int deviceIndex, String keyId) { + if (deviceIndex < 0 || deviceIndex >= snapshot.devices().size()) { + return ImmutableList.of(); + } + String uuid = snapshot.devices().get(deviceIndex).deviceId(); + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.valuesForUuid(uuid) : ImmutableList.of(); + } + + private static final int[] EMPTY_INT_ARRAY = new int[0]; + } + /** Default empty implementation of {@link OverlayView}. */ final class EmptyOverlayView implements OverlayView { static final EmptyOverlayView INSTANCE = new EmptyOverlayView(); diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/AtsOneFleetDataSource.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/AtsOneFleetDataSource.java index 759062012e..cf446d59e2 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/AtsOneFleetDataSource.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/AtsOneFleetDataSource.java @@ -84,6 +84,11 @@ public ListenableFuture pull() { executor); } + @Override + public ListenableFuture pullDimension(String keyId) { + return labInfoFleetPuller.pullDimension(keyId); + } + /** Enumerates the device ids in the lab query result, in lab then device order. */ private static ImmutableList deviceIds(LabQueryResult labData) { if (!labData.hasLabView()) { diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/BUILD index d94dfba4f7..c4cab424db 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/BUILD +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/BUILD @@ -20,16 +20,28 @@ package( default_visibility = ["//src/devtools/mobileharness/fe/v6:visibility"], ) +java_library( + name = "dimension_overlay_raw", + srcs = ["DimensionOverlayRaw.java"], + deps = [ + "//src/java/com/google/devtools/mobileharness/shared/util/auto:auto_value", + "@maven//:com_google_guava_guava", + ], +) + java_library( name = "lab_info_fleet_puller", srcs = ["LabInfoFleetPuller.java"], deps = [ + ":dimension_overlay_raw", + "//src/devtools/mobileharness/api/model/proto:device_java_proto", "//src/devtools/mobileharness/api/query/proto:lab_query_java_proto", "//src/devtools/mobileharness/shared/labinfo/proto:lab_info_service_java_proto", "//src/java/com/google/devtools/mobileharness/fe/v6/service/shared/providers:lab_info_provider", "//src/java/com/google/devtools/mobileharness/fe/v6/service/util", "//src/java/com/google/devtools/mobileharness/shared/util/logging:google_logger", "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", "@maven//:javax_inject_jsr330_api", ], ) @@ -38,6 +50,7 @@ java_library( name = "fleet_data_source", srcs = ["FleetDataSource.java"], deps = [ + ":dimension_overlay_raw", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_raw_data", "@maven//:com_google_guava_guava", @@ -48,6 +61,7 @@ java_library( name = "ats_one_fleet_data_source", srcs = ["AtsOneFleetDataSource.java"], deps = [ + ":dimension_overlay_raw", ":fleet_data_source", ":lab_info_fleet_puller", "//src/devtools/mobileharness/api/deviceconfig/proto:device_java_proto", diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/DimensionOverlayRaw.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/DimensionOverlayRaw.java new file mode 100644 index 0000000000..092da49365 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/DimensionOverlayRaw.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.pull; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +/** Raw dimension data pulled from a data source for one on-demand long-tail dimension. */ +@AutoValue +public abstract class DimensionOverlayRaw { + + /** The namespaced key id (e.g. "dim::carrier"). */ + public abstract String keyId(); + + /** Map from device UUID to the list of display-cased values for this dimension. */ + public abstract ImmutableMap> uuidToValues(); + + public static DimensionOverlayRaw create( + String keyId, ImmutableMap> uuidToValues) { + return new AutoValue_DimensionOverlayRaw(keyId, uuidToValues); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/FleetDataSource.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/FleetDataSource.java index 0b892010ed..ecf104869f 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/FleetDataSource.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/FleetDataSource.java @@ -46,4 +46,10 @@ public interface FleetDataSource { * keeps the previously published snapshot for this fleet. */ ListenableFuture pull(); + + /** + * Starts an on-demand pull of a single dimension's values for this fleet and returns a future for + * the raw overlay data. + */ + ListenableFuture pullDimension(String keyId); } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/LabInfoFleetPuller.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/LabInfoFleetPuller.java index 3c0aa03b23..5562913b54 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/LabInfoFleetPuller.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/LabInfoFleetPuller.java @@ -19,19 +19,27 @@ import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.flogger.FluentLogger; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.devtools.mobileharness.api.model.proto.Device.DeviceDimension; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.DeviceInfo; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabData; import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQuery; import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQueryResult; import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.Page; import com.google.devtools.mobileharness.fe.v6.service.shared.providers.LabInfoProvider; import com.google.devtools.mobileharness.fe.v6.service.util.UniverseScope; import com.google.devtools.mobileharness.shared.labinfo.proto.LabInfoServiceProto.GetLabInfoRequest; +import com.google.protobuf.FieldMask; +import java.util.LinkedHashSet; +import java.util.Set; import javax.inject.Inject; /** - * Pulls the full fleet from {@code LabInfoService} for the search index refresh cycle. + * Pulls the full fleet or on-demand single dimensions from {@code LabInfoService}. * *

Unlike the per-entity detail-page reads, this asks for the whole fleet in one call: no filter * (every lab and device) and no page limit. It accepts the master's cached data, which is enough @@ -58,6 +66,12 @@ public final class LabInfoFleetPuller { .setPage(Page.newBuilder().setLimit(0)) .build(); + private static final FieldMask SINGLE_DIM_FIELD_MASK = + FieldMask.newBuilder() + .addPaths("device_locator.id") + .addPaths("device_feature.composite_dimension") + .build(); + private final LabInfoProvider labInfoProvider; @Inject @@ -70,12 +84,18 @@ public final class LabInfoFleetPuller { * LabQueryResult}. */ public ListenableFuture pull() { + return pull(UniverseScope.SELF); + } + + /** Starts a full-fleet pull for the specified universe. */ + public ListenableFuture pull(UniverseScope universeScope) { logger.atInfo().log( - "Issuing GetLabInfo to the master (SELF universe): full fleet, no page limit, cached" - + " data."); + "Issuing GetLabInfo to the master (%s universe): full fleet, no page limit, cached" + + " data.", + universeScope); Stopwatch stopwatch = Stopwatch.createStarted(); return Futures.transform( - labInfoProvider.getLabInfoAsync(FULL_FLEET_REQUEST, UniverseScope.SELF), + labInfoProvider.getLabInfoAsync(FULL_FLEET_REQUEST, universeScope), response -> { LabQueryResult result = response.getLabQueryResult(); logger.atInfo().log( @@ -85,4 +105,81 @@ public ListenableFuture pull() { }, directExecutor()); } + + /** Starts an on-demand single dimension pull from the specified universe. */ + public ListenableFuture pullDimension( + String keyId, UniverseScope universeScope) { + String dimName = keyId.startsWith("dim::") ? keyId.substring("dim::".length()) : keyId; + + GetLabInfoRequest request = + GetLabInfoRequest.newBuilder() + .setLabQuery( + LabQuery.newBuilder() + .setLabViewRequest(LabQuery.LabViewRequest.getDefaultInstance()) + .setMask( + LabQuery.Mask.newBuilder() + .setDeviceInfoMask( + LabQuery.Mask.DeviceInfoMask.newBuilder() + .setFieldMask(SINGLE_DIM_FIELD_MASK) + .setSupportedDimensionsMask( + LabQuery.Mask.DeviceInfoMask.DimensionsMask.newBuilder() + .addDimensionNames(dimName)) + .setRequiredDimensionsMask( + LabQuery.Mask.DeviceInfoMask.DimensionsMask.newBuilder() + .addDimensionNames(dimName))))) + .setPage(Page.newBuilder().setLimit(0)) + .build(); + + Stopwatch stopwatch = Stopwatch.createStarted(); + return Futures.transform( + labInfoProvider.getLabInfoAsync(request, universeScope), + response -> { + LabQueryResult result = response.getLabQueryResult(); + ImmutableMap.Builder> uuidToValues = ImmutableMap.builder(); + + if (result.hasLabView()) { + for (LabData labData : result.getLabView().getLabDataList()) { + for (DeviceInfo deviceInfo : labData.getDeviceList().getDeviceInfoList()) { + String uuid = deviceInfo.getDeviceLocator().getId(); + if (uuid.isEmpty()) { + continue; + } + Set values = new LinkedHashSet<>(); + for (DeviceDimension dim : + deviceInfo + .getDeviceFeature() + .getCompositeDimension() + .getSupportedDimensionList()) { + if (dim.getName().equals(dimName) && !dim.getValue().isEmpty()) { + values.add(dim.getValue()); + } + } + for (DeviceDimension dim : + deviceInfo + .getDeviceFeature() + .getCompositeDimension() + .getRequiredDimensionList()) { + if (dim.getName().equals(dimName) && !dim.getValue().isEmpty()) { + values.add(dim.getValue()); + } + } + if (!values.isEmpty()) { + uuidToValues.put(uuid, ImmutableList.copyOf(values)); + } + } + } + } + + logger.atInfo().log( + "GetLabInfo on-demand dim '%s' returned %d devices in %d ms.", + keyId, uuidToValues.buildKeepingLast().size(), stopwatch.elapsed().toMillis()); + return DimensionOverlayRaw.create(keyId, uuidToValues.buildKeepingLast()); + }, + directExecutor()); + } + + /** Starts an on-demand single dimension pull from the self universe. */ + public ListenableFuture pullDimension(String keyId) { + return pullDimension(keyId, UniverseScope.SELF); + } } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/BUILD index ff177bdd49..935db8a89b 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/BUILD +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/BUILD @@ -43,11 +43,14 @@ java_library( ":search_corpus", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_common_java_proto", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:composite_fleet_index", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:composite_postings", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:device_record", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:device_value_extractor", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_search_keys", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:overlay_view", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:postings", "@maven//:com_google_code_findbugs_jsr305", "@maven//:com_google_guava_guava", diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/DeviceCorpus.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/DeviceCorpus.java index fc9702a659..c18ad3cf54 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/DeviceCorpus.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/DeviceCorpus.java @@ -16,6 +16,8 @@ package com.google.devtools.mobileharness.fe.v6.service.search.query; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -23,11 +25,15 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.Column; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetUtilization; import com.google.devtools.mobileharness.fe.v6.service.proto.search.SearchEntity; +import com.google.devtools.mobileharness.fe.v6.service.proto.search.TextCell; +import com.google.devtools.mobileharness.fe.v6.service.search.index.CompositeFleetIndex; +import com.google.devtools.mobileharness.fe.v6.service.search.index.CompositePostings; import com.google.devtools.mobileharness.fe.v6.service.search.index.DeviceRecord; import com.google.devtools.mobileharness.fe.v6.service.search.index.DeviceValueExtractor; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndex; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSearchKeys; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; +import com.google.devtools.mobileharness.fe.v6.service.search.index.OverlayView; import com.google.devtools.mobileharness.fe.v6.service.search.index.Postings; import java.util.List; import java.util.Optional; @@ -38,8 +44,7 @@ * *

Records are the snapshot's devices, identified by device UUID. The value projection delegates * to {@link DeviceValueExtractor} (lowercased sets) and {@link FleetCellMapper} (display values, - * headers, typed cells), so it mirrors exactly what the device index builder recorded. Utilization - * is the device idle / busy / other bucketing, so a device group carries a utilization breakdown. + * headers, typed cells), falling back to {@link OverlayView} for on-demand long-tail dimensions. */ public final class DeviceCorpus implements SearchCorpus { @@ -59,20 +64,32 @@ public final class DeviceCorpus implements SearchCorpus { "FASTBOOTDMODE"); private final FleetSnapshot snapshot; + private final FleetIndex index; private final Postings postings; + private final OverlayView overlayView; @Nullable private final ScenarioCuration curation; private final FleetCellMapper cellMapper = new FleetCellMapper(); public DeviceCorpus( - FleetSnapshot snapshot, Postings postings, @Nullable ScenarioCuration curation) { - this.snapshot = snapshot; - this.postings = postings; + FleetSnapshot snapshot, + Postings postings, + @Nullable ScenarioCuration curation, + OverlayView overlayView) { + this.snapshot = checkNotNull(snapshot); + this.overlayView = checkNotNull(overlayView); + this.index = new CompositeFleetIndex(snapshot.index(), overlayView); + this.postings = new CompositePostings(postings, overlayView); this.curation = curation; } + public DeviceCorpus( + FleetSnapshot snapshot, Postings postings, @Nullable ScenarioCuration curation) { + this(snapshot, postings, curation, OverlayView.empty()); + } + @Override public FleetIndex index() { - return snapshot.index(); + return index; } @Override @@ -107,21 +124,34 @@ public boolean plainValueKey(String keyId) { @Override public ImmutableSet valuesForKey(int index, String keyId) { + if (overlayView.containsKey(keyId)) { + return overlayView.valuesForKey(index, keyId); + } return DeviceValueExtractor.valuesForKey(snapshot.devices().get(index), keyId); } @Override public ImmutableList displayValues(int index, String keyId) { + if (overlayView.containsKey(keyId)) { + return overlayView.displayValues(index, keyId); + } return FleetCellMapper.displayValues(snapshot.devices().get(index), keyId, snapshot); } @Override public Column column(String keyId) { - return cellMapper.column(keyId, snapshot); + return Column.newBuilder().setKey(keyId).setDisplayName(index.displayName(keyId)).build(); } @Override public Cell cell(int index, String keyId) { + if (overlayView.containsKey(keyId)) { + return Cell.newBuilder() + .setText( + TextCell.newBuilder() + .setValue(String.join(", ", overlayView.displayValues(index, keyId)))) + .build(); + } return cellMapper.cell(keyId, snapshot.devices().get(index), snapshot); } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSuggester.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSuggester.java index 8d072ff6ca..6b363a287a 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSuggester.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSuggester.java @@ -432,12 +432,38 @@ private List suggestKv(Context context, String keyToken, String rawValue, String value = Ascii.toLowerCase(stripQuotes(rawValue.trim())); for (String keyId : keyIds) { + boolean hadMatches = false; for (Match match : matchValues(context.index(), keyId, value, /* allowContains= */ true)) { Cand cand = condition(context, keyId, match.value(), match.tier(), exclude); if (cand != null) { out.add(cand); + hadMatches = true; } } + // Cold long-tail fallback: if the key is valid but has no index entries in core/overlay, + // and is not already in active filters, emit a ready-to-apply filter condition with no count. + if (!hadMatches && !value.isEmpty() && !context.activeKeys().contains(keyId)) { + String display = context.index().displayName(keyId); + String op = exclude ? "is not" : "is"; + ImmutableList mainText = segments(display + " " + op + " ", rawValue.trim()); + Filter filter = + Filter.newBuilder() + .setKey(keyId) + .setSimple( + SimpleMatch.newBuilder() + .addValues(FilterValue.newBuilder().setValue(rawValue.trim())) + .setNegated(exclude)) + .build(); + FleetSuggestion.Builder builder = + FleetSuggestion.newBuilder() + .setLabel("Add filter") + .addAllMainText(mainText) + .setApplyFilter(applyFilter(context.index(), keyId, filter)); + Cand cand = new Cand(Kind.CONDITION, keyId, 1.0, builder, mainTextString(mainText)); + cand.needsCount = false; + cand.noCount = true; + out.add(cand); + } } return out; } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD index b785d08052..36fad5a8af 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD @@ -48,6 +48,21 @@ java_library( ], ) +java_library( + name = "dimension_overlay_store", + srcs = ["DimensionOverlayStore.java"], + deps = [ + ":fleet_snapshot_store", + "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:dimension_overlay", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:dimension_overlay_raw", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:fleet_data_source", + "@maven//:com_google_guava_guava", + "@maven//:javax_inject_jsr330_api", + ], +) + java_library( name = "fleet_search_data_module", srcs = ["FleetSearchDataModule.java"], diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStore.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStore.java new file mode 100644 index 0000000000..a9d95f783b --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStore.java @@ -0,0 +1,150 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.refresh; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.Futures.immediateFuture; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import com.google.devtools.mobileharness.fe.v6.service.proto.search.Fleet; +import com.google.devtools.mobileharness.fe.v6.service.search.index.DimensionOverlay; +import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.DimensionOverlayRaw; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.FleetDataSource; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; +import javax.inject.Inject; +import javax.inject.Singleton; + +/** + * Thread-safe store for on-demand {@link DimensionOverlay} instances across fleets. + * + *

Asynchronously loads cold dimensions via {@link FleetDataSource#pullDimension(String)} with + * in-flight deduplication and returns strong-reference maps for query execution. Overlays are + * cached in a bounded per-fleet LRU cache (capacity 50, TTL 30m). + */ +@Singleton +public final class DimensionOverlayStore { + + private static final long MAX_CACHE_SIZE = 50L; + private static final Duration CACHE_TTL = Duration.ofMinutes(30); + + private final Map dataSources; + private final FleetSnapshotStore snapshotStore; + private final ConcurrentMap> memoryCaches = + new ConcurrentHashMap<>(); + private final ConcurrentMap>> + inFlight = new ConcurrentHashMap<>(); + + @Inject + DimensionOverlayStore(Map dataSources, FleetSnapshotStore snapshotStore) { + this.dataSources = checkNotNull(dataSources); + this.snapshotStore = checkNotNull(snapshotStore); + } + + /** + * Asynchronously loads all requested overlay keys for the given fleet and returns a strong + * reference map of loaded overlays. Keys that are already cached return immediately. + */ + public ListenableFuture> loadOverlaysAsync( + Fleet fleet, Set keyIds, Executor executor) { + if (keyIds.isEmpty()) { + return immediateFuture(ImmutableMap.of()); + } + + FleetDataSource dataSource = dataSources.get(fleet); + if (dataSource == null) { + return immediateFuture(ImmutableMap.of()); + } + + Cache cache = + memoryCaches.computeIfAbsent( + fleet, + f -> + CacheBuilder.newBuilder() + .maximumSize(MAX_CACHE_SIZE) + .expireAfterWrite(CACHE_TTL) + .build()); + ConcurrentMap> inFlightMap = + inFlight.computeIfAbsent(fleet, f -> new ConcurrentHashMap<>()); + + List>> futures = new ArrayList<>(); + + for (String keyId : keyIds) { + DimensionOverlay cached = cache.getIfPresent(keyId); + if (cached != null) { + futures.add(immediateFuture(Map.entry(keyId, cached))); + continue; + } + + ListenableFuture pullFuture = inFlightMap.get(keyId); + if (pullFuture == null) { + SettableFuture settable = SettableFuture.create(); + ListenableFuture existing = inFlightMap.putIfAbsent(keyId, settable); + if (existing != null) { + pullFuture = existing; + } else { + pullFuture = settable; + ListenableFuture rawFuture = dataSource.pullDimension(keyId); + Futures.addCallback( + rawFuture, + new FutureCallback<>() { + @Override + public void onSuccess(DimensionOverlayRaw raw) { + try { + FleetSnapshot snapshot = snapshotStore.get(fleet); + DimensionOverlay overlay = DimensionOverlay.create(raw, snapshot); + cache.put(keyId, overlay); + inFlightMap.remove(keyId); + settable.set(overlay); + } catch (Throwable t) { + inFlightMap.remove(keyId); + settable.setException(t); + } + } + + @Override + public void onFailure(Throwable t) { + inFlightMap.remove(keyId); + settable.setException(t); + } + }, + executor); + } + } + + futures.add(Futures.transform(pullFuture, overlay -> Map.entry(keyId, overlay), executor)); + } + + return Futures.transform( + Futures.allAsList(futures), + entries -> ImmutableMap.builder().putAll(entries).buildOrThrow(), + executor); + } +} diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/BUILD b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/BUILD index b477eba96a..40843d9b09 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/BUILD +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/BUILD @@ -32,6 +32,7 @@ java_library( "//src/java/com/google/devtools/mobileharness/fe/v6/service/search:search_service_logic_impl", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index_builder", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:fleet_data_source", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:scenario_curation_module", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:fleet_snapshot_store", "//src/javatests/com/google/devtools/mobileharness/builddefs:truth", diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImplTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImplTest.java index 025fb28095..3913fd338e 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImplTest.java +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImplTest.java @@ -44,11 +44,13 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.SearchEntity; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndexBuilder; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.FleetDataSource; import com.google.devtools.mobileharness.fe.v6.service.search.query.ScenarioCurationModule; import com.google.devtools.mobileharness.fe.v6.service.search.refresh.FleetSnapshotStore; import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; +import com.google.inject.multibindings.MapBinder; import java.time.Instant; import org.junit.Before; import org.junit.Test; @@ -84,6 +86,7 @@ public void setUp() { @Override protected void configure() { bind(ListeningExecutorService.class).toInstance(newDirectExecutorService()); + MapBinder.newMapBinder(binder(), Fleet.class, FleetDataSource.class); } }); FleetSnapshotStore store = injector.getInstance(FleetSnapshotStore.class); diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD index c9450ad8dc..26dc23d8da 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD @@ -31,6 +31,7 @@ java_library( "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:composite_fleet_index", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:composite_postings", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:device_enrichment", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:dimension_overlay", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index_builder", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_raw_data", @@ -41,6 +42,7 @@ java_library( "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:overlay_view", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:postings", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:value_key_pair", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:dimension_overlay_raw", "//src/javatests/com/google/devtools/mobileharness/builddefs:truth", "@maven//:com_google_guava_guava", "@maven//:junit_junit", diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlayTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlayTest.java new file mode 100644 index 0000000000..2bae2c51c8 --- /dev/null +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlayTest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.DimensionOverlayRaw; +import java.time.Instant; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link DimensionOverlay}. */ +@RunWith(JUnit4.class) +public final class DimensionOverlayTest { + + @Test + public void create_and_bind_reindexesAccurately() { + DimensionOverlayRaw raw = + DimensionOverlayRaw.create( + "dim::carrier", + ImmutableMap.of( + "uuid-1", ImmutableList.of("Verizon"), + "uuid-2", ImmutableList.of("T-Mobile", "AT&T"), + "uuid-3", ImmutableList.of("Verizon"))); + + FleetSnapshot snapshotA = + FleetSnapshot.builder() + .setBuildTime(Instant.ofEpochSecond(1000)) + .setDevices(ImmutableList.of()) + .setHosts(ImmutableList.of()) + .setIndex(CoreFleetIndex.empty()) + .setHostIndex(CoreFleetIndex.empty()) + .setUuidToIndex(ImmutableMap.of("uuid-1", 0, "uuid-2", 1, "uuid-3", 2)) + .build(); + + DimensionOverlay overlay = DimensionOverlay.create(raw, snapshotA); + + assertThat(overlay.keyId()).isEqualTo("dim::carrier"); + assertThat(overlay.sortedValues()).containsExactly("at&t", "t-mobile", "verizon").inOrder(); + assertThat(overlay.valueCounts()).containsEntry("verizon", 2); + assertThat(overlay.valueDisplays()).containsEntry("verizon", "Verizon"); + + // Check postings bound to Snapshot A + DimensionOverlay.SnapshotBoundPostings boundA = overlay.bind(snapshotA); + assertThat(boundA.get("verizon")).asList().containsExactly(0, 2).inOrder(); + assertThat(boundA.get("t-mobile")).asList().containsExactly(1); + assertThat(boundA.get("at&t")).asList().containsExactly(1); + + // Snapshot B: uuid-1 moved to index 5, uuid-3 moved to index 1, uuid-2 is decommissioned + // (absent) + FleetSnapshot snapshotB = + FleetSnapshot.builder() + .setBuildTime(Instant.ofEpochSecond(2000)) + .setDevices(ImmutableList.of()) + .setHosts(ImmutableList.of()) + .setIndex(CoreFleetIndex.empty()) + .setHostIndex(CoreFleetIndex.empty()) + .setUuidToIndex(ImmutableMap.of("uuid-1", 5, "uuid-3", 1)) + .build(); + + DimensionOverlay.SnapshotBoundPostings boundB = overlay.bind(snapshotB); + assertThat(boundB.get("verizon")).asList().containsExactly(1, 5).inOrder(); + assertThat(boundB.get("t-mobile")).isEmpty(); // uuid-2 was dropped safely + } +} diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD index 3f9314fed4..6dbcaa6e56 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD @@ -28,13 +28,16 @@ java_library( "//src/devtools/mobileharness/api/model/proto:device_java_proto", "//src/devtools/mobileharness/api/query/proto:lab_query_java_proto", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:dimension_overlay", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index_builder", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_raw_data", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:host_record", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:lazy_postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:dimension_overlay_raw", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:fleet_data_source", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:dimension_overlay_store", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:fleet_data_refresher", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:fleet_snapshot_store", "//src/java/com/google/devtools/mobileharness/shared/util/concurrent:thread_pools", diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStoreTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStoreTest.java new file mode 100644 index 0000000000..ff98ddee70 --- /dev/null +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStoreTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.refresh; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.SettableFuture; +import com.google.devtools.mobileharness.fe.v6.service.proto.search.Fleet; +import com.google.devtools.mobileharness.fe.v6.service.search.index.CoreFleetIndex; +import com.google.devtools.mobileharness.fe.v6.service.search.index.DimensionOverlay; +import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetRawData; +import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.DimensionOverlayRaw; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.FleetDataSource; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link DimensionOverlayStore}. */ +@RunWith(JUnit4.class) +public final class DimensionOverlayStoreTest { + + private final ListeningExecutorService executor = newDirectExecutorService(); + private final FleetSnapshotStore snapshotStore = new FleetSnapshotStore(); + private FakeDataSource dataSource; + private DimensionOverlayStore overlayStore; + + @Before + public void setUp() { + FleetSnapshot snapshot = + FleetSnapshot.builder() + .setBuildTime(Instant.ofEpochSecond(1_000)) + .setDevices(ImmutableList.of()) + .setHosts(ImmutableList.of()) + .setIndex(CoreFleetIndex.empty()) + .setHostIndex(CoreFleetIndex.empty()) + .setUuidToIndex(ImmutableMap.of("dev-1", 0, "dev-2", 1)) + .build(); + snapshotStore.publish(Fleet.FLEET_SELF, snapshot); + + dataSource = new FakeDataSource(); + overlayStore = + new DimensionOverlayStore(ImmutableMap.of(Fleet.FLEET_SELF, dataSource), snapshotStore); + } + + @Test + public void loadOverlaysAsync_emptyKeys_returnsImmediately() throws Exception { + ImmutableMap result = + overlayStore.loadOverlaysAsync(Fleet.FLEET_SELF, ImmutableSet.of(), executor).get(); + + assertThat(result).isEmpty(); + assertThat(dataSource.pullCount.get()).isEqualTo(0); + } + + @Test + public void loadOverlaysAsync_coldKey_pullsFromDataSourceAndCaches() throws Exception { + dataSource.rawResult = + DimensionOverlayRaw.create( + "dim::carrier", ImmutableMap.of("dev-1", ImmutableList.of("Verizon"))); + + ImmutableMap result = + overlayStore + .loadOverlaysAsync(Fleet.FLEET_SELF, ImmutableSet.of("dim::carrier"), executor) + .get(); + + assertThat(result.keySet()).containsExactly("dim::carrier"); + assertThat(result.get("dim::carrier").valueCounts()).containsEntry("verizon", 1); + assertThat(dataSource.pullCount.get()).isEqualTo(1); + + // Second call hits cache; does not pull again + ImmutableMap secondResult = + overlayStore + .loadOverlaysAsync(Fleet.FLEET_SELF, ImmutableSet.of("dim::carrier"), executor) + .get(); + + assertThat(secondResult.keySet()).containsExactly("dim::carrier"); + assertThat(dataSource.pullCount.get()).isEqualTo(1); + } + + @Test + public void loadOverlaysAsync_concurrentCallsForSameKey_deduplicates() throws Exception { + SettableFuture inFlight = SettableFuture.create(); + dataSource.setPendingFuture(inFlight); + + ListenableFuture> future1 = + overlayStore.loadOverlaysAsync(Fleet.FLEET_SELF, ImmutableSet.of("dim::carrier"), executor); + ListenableFuture> future2 = + overlayStore.loadOverlaysAsync(Fleet.FLEET_SELF, ImmutableSet.of("dim::carrier"), executor); + + assertThat(dataSource.pullCount.get()).isEqualTo(1); + + // Complete the in-flight pull + inFlight.set( + DimensionOverlayRaw.create( + "dim::carrier", ImmutableMap.of("dev-1", ImmutableList.of("Verizon")))); + + assertThat(future1.get()).containsKey("dim::carrier"); + assertThat(future2.get()).containsKey("dim::carrier"); + assertThat(dataSource.pullCount.get()).isEqualTo(1); + } + + private static final class FakeDataSource implements FleetDataSource { + final AtomicInteger pullCount = new AtomicInteger(0); + volatile DimensionOverlayRaw rawResult = + DimensionOverlayRaw.create("dim::empty", ImmutableMap.of()); + volatile SettableFuture pendingFuture = null; + + void setPendingFuture(SettableFuture pending) { + this.pendingFuture = pending; + } + + @Override + public Fleet fleet() { + return Fleet.FLEET_SELF; + } + + @Override + public ListenableFuture pull() { + throw new UnsupportedOperationException(); + } + + @Override + public ListenableFuture pullDimension(String keyId) { + pullCount.incrementAndGet(); + if (pendingFuture != null) { + return pendingFuture; + } + return immediateFuture(rawResult); + } + } +} diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetDataRefresherTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetDataRefresherTest.java index d4b4aa8705..06ea1836b1 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetDataRefresherTest.java +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetDataRefresherTest.java @@ -32,6 +32,7 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.Fleet; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndexBuilder; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetRawData; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.DimensionOverlayRaw; import com.google.devtools.mobileharness.fe.v6.service.search.pull.FleetDataSource; import com.google.devtools.mobileharness.shared.util.concurrent.ThreadPools; import com.google.inject.Guice; @@ -150,5 +151,10 @@ public Fleet fleet() { public ListenableFuture pull() { return result; } + + @Override + public ListenableFuture pullDimension(String keyId) { + return immediateFuture(DimensionOverlayRaw.create(keyId, ImmutableMap.of())); + } } }