From cd888a699d1e1c832f63ad587f0146b1a7094633 Mon Sep 17 00:00:00 2001 From: OmniLab Team Date: Thu, 20 Aug 2026 10:24:53 -0700 Subject: [PATCH] Internal change PiperOrigin-RevId: 967906512 --- .../mobileharness/fe/v6/service/search/BUILD | 3 + .../search/SearchServiceLogicImpl.java | 114 +++++++-- .../fe/v6/service/search/index/BUILD | 55 ++++- .../search/index/CompositeFleetIndex.java | 100 ++++++++ .../search/index/CompositePostings.java | 54 +++++ .../service/search/index/CoreFleetIndex.java | 126 ++++++++++ .../search/index/DimensionOverlay.java | 216 ++++++++++++++++++ .../v6/service/search/index/FleetIndex.java | 124 ++++------ .../search/index/FleetIndexBuilder.java | 21 +- .../service/search/index/FleetSearchKeys.java | 15 ++ .../service/search/index/FleetSnapshot.java | 13 +- .../v6/service/search/index/LazyPostings.java | 20 +- .../v6/service/search/index/OverlayView.java | 208 +++++++++++++++++ .../fe/v6/service/search/index/Postings.java | 35 +++ .../search/pull/AtsOneFleetDataSource.java | 5 + .../fe/v6/service/search/pull/BUILD | 15 ++ .../search/pull/DimensionOverlayRaw.java | 37 +++ .../service/search/pull/FleetDataSource.java | 6 + .../search/pull/LabInfoFleetPuller.java | 136 ++++++++++- .../fe/v6/service/search/query/BUILD | 17 +- .../v6/service/search/query/DeviceCorpus.java | 50 +++- .../service/search/query/FleetCellMapper.java | 20 +- .../search/query/FleetChipResolver.java | 32 +-- .../search/query/FleetColumnCataloger.java | 28 +-- .../search/query/FleetFilterEngine.java | 23 +- .../search/query/FleetGroupSearcher.java | 3 +- .../query/FleetPromotedKeysProvider.java | 13 +- .../query/FleetSearchConfigProvider.java | 18 +- .../service/search/query/FleetSuggester.java | 65 +++--- .../search/query/FleetValueLister.java | 24 +- .../service/search/query/HostCellMapper.java | 21 +- .../v6/service/search/query/HostCorpus.java | 8 +- .../v6/service/search/query/SearchCorpus.java | 4 +- .../fe/v6/service/search/refresh/BUILD | 15 ++ .../search/refresh/DimensionOverlayStore.java | 137 +++++++++++ .../search/schema/AtsDeviceKeyRegistry.java | 33 +++ .../service/search/schema/AtsDeviceKeys.java | 42 ++++ .../search/schema/AtsHostKeyRegistry.java | 33 +++ .../fe/v6/service/search/schema/BUILD | 109 +++++++++ .../search/schema/DeviceInfoSource.java | 126 ++++++++++ .../search/schema/DeviceKeyDescriptor.java | 96 ++++++++ .../search/schema/DeviceKeyRegistry.java | 174 ++++++++++++++ .../v6/service/search/schema/DeviceKeys.java | 186 +++++++++++++++ .../search/schema/HostKeyDescriptor.java | 77 +++++++ .../search/schema/HostKeyRegistry.java | 119 ++++++++++ .../fe/v6/service/search/schema/HostKeys.java | 106 +++++++++ .../v6/service/search/schema/KeyDisplay.java | 54 +++++ .../service/search/schema/LabInfoSource.java | 105 +++++++++ .../mobileharness/fe/v6/service/search/BUILD | 1 + .../search/SearchServiceLogicImplTest.java | 3 + .../fe/v6/service/search/index/BUILD | 8 + .../search/index/CompositeFleetIndexTest.java | 178 +++++++++++++++ .../search/index/CompositePostingsTest.java | 155 +++++++++++++ .../search/index/DimensionOverlayTest.java | 82 +++++++ .../search/index/FleetIndexBuilderTest.java | 54 +++-- .../fe/v6/service/search/refresh/BUILD | 3 + .../refresh/DimensionOverlayStoreTest.java | 155 +++++++++++++ .../refresh/FleetDataRefresherTest.java | 6 + .../refresh/FleetSnapshotStoreTest.java | 10 +- .../fe/v6/service/search/schema/BUILD | 47 ++++ .../search/schema/DeviceKeyRegistryTest.java | 184 +++++++++++++++ .../search/schema/HostKeyRegistryTest.java | 101 ++++++++ 62 files changed, 3666 insertions(+), 362 deletions(-) create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositeFleetIndex.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositePostings.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CoreFleetIndex.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlay.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/OverlayView.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/Postings.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/DimensionOverlayRaw.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStore.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsDeviceKeyRegistry.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsDeviceKeys.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsHostKeyRegistry.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/BUILD create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceInfoSource.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyDescriptor.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyRegistry.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeys.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyDescriptor.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyRegistry.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeys.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/KeyDisplay.java create mode 100644 src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/LabInfoSource.java create mode 100644 src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositeFleetIndexTest.java create mode 100644 src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositePostingsTest.java create mode 100644 src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlayTest.java create mode 100644 src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStoreTest.java create mode 100644 src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/BUILD create mode 100644 src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyRegistryTest.java create mode 100644 src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyRegistryTest.java diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/BUILD index be993a96d6..651b0ba2e0 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/BUILD +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/BUILD @@ -35,7 +35,9 @@ java_library( deps = [ ":search_service_logic", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:dimension_overlay", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:overlay_view", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:device_corpus", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:fleet_chip_resolver", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:fleet_column_cataloger", @@ -48,6 +50,7 @@ java_library( "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:host_corpus", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:scenario_curation", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:search_corpus", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:dimension_overlay_store", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:fleet_snapshot_store", "@maven//:com_google_guava_guava", "@maven//:javax_inject_jsr330_api", diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImpl.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImpl.java index 8b6c33fa04..5765c9685c 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImpl.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImpl.java @@ -18,9 +18,12 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.devtools.mobileharness.fe.v6.service.proto.search.Filter; import com.google.devtools.mobileharness.fe.v6.service.proto.search.Fleet; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetChipResolverRequest; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetChipResolverResponse; @@ -42,7 +45,9 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetValueListRequest; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetValueListResponse; import com.google.devtools.mobileharness.fe.v6.service.proto.search.SearchEntity; +import com.google.devtools.mobileharness.fe.v6.service.search.index.DimensionOverlay; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; +import com.google.devtools.mobileharness.fe.v6.service.search.index.OverlayView; import com.google.devtools.mobileharness.fe.v6.service.search.query.DeviceCorpus; import com.google.devtools.mobileharness.fe.v6.service.search.query.FleetChipResolver; import com.google.devtools.mobileharness.fe.v6.service.search.query.FleetColumnCataloger; @@ -55,8 +60,11 @@ import com.google.devtools.mobileharness.fe.v6.service.search.query.HostCorpus; import com.google.devtools.mobileharness.fe.v6.service.search.query.ScenarioCuration; import com.google.devtools.mobileharness.fe.v6.service.search.query.SearchCorpus; +import com.google.devtools.mobileharness.fe.v6.service.search.refresh.DimensionOverlayStore; import com.google.devtools.mobileharness.fe.v6.service.search.refresh.FleetSnapshotStore; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import javax.inject.Inject; import javax.inject.Singleton; @@ -82,6 +90,7 @@ public final class SearchServiceLogicImpl implements SearchServiceLogic { private final ListeningExecutorService executor; private final FleetSnapshotStore store; + private final DimensionOverlayStore overlayStore; private final Map curations; private final FleetSearchConfigProvider searchConfigProvider; private final FleetFlatSearcher flatSearcher; @@ -96,6 +105,7 @@ public final class SearchServiceLogicImpl implements SearchServiceLogic { SearchServiceLogicImpl( ListeningExecutorService executor, FleetSnapshotStore store, + DimensionOverlayStore overlayStore, Map curations, FleetSearchConfigProvider searchConfigProvider, FleetFlatSearcher flatSearcher, @@ -107,6 +117,7 @@ public final class SearchServiceLogicImpl implements SearchServiceLogic { FleetColumnCataloger columnCataloger) { this.executor = executor; this.store = store; + this.overlayStore = overlayStore; this.curations = curations; this.searchConfigProvider = searchConfigProvider; this.flatSearcher = flatSearcher; @@ -134,12 +145,18 @@ public ListenableFuture getFleetSearchConfig( @Override public ListenableFuture searchFleet(FleetSearchRequest request) { - return Futures.submit(() -> searchFleetSync(request), executor); + Fleet fleet = normalize(request.getFleet()); + Set referencedKeys = extractReferencedKeys(request); + return Futures.transformAsync( + overlayStore.loadOverlaysAsync(fleet, referencedKeys, executor), + overlays -> Futures.submit(() -> searchFleetSync(request, overlays), executor), + executor); } - private FleetSearchResults searchFleetSync(FleetSearchRequest request) { + private FleetSearchResults searchFleetSync( + FleetSearchRequest request, ImmutableMap overlays) { Fleet fleet = normalize(request.getFleet()); - SearchCorpus corpus = corpus(fleet, request.getEntity()); + SearchCorpus corpus = corpus(fleet, request.getEntity(), overlays); return switch (request.getViewCase()) { case FLAT -> { FleetFlatView flat = request.getFlat(); @@ -174,8 +191,6 @@ private FleetSearchResults searchFleetSync(FleetSearchRequest request) { expand.getPageToken()); yield FleetSearchResults.newBuilder().setFlat(results).build(); } - // A request with no view selects no results shape, so return an empty result rather than - // guessing a view. case VIEW_NOT_SET -> FleetSearchResults.getDefaultInstance(); }; } @@ -196,9 +211,6 @@ public ListenableFuture resolveFleetChips( FleetChipResolverRequest request) { return Futures.submit( () -> { - // Chip resolution is stateless: the request carries no fleet, and the resolver reads only - // key display names and value casing, which are the same across fleets. Read the self - // snapshot. FleetSnapshot snapshot = store.get(Fleet.FLEET_SELF); return chipResolver.resolve(snapshot, request); }, @@ -207,12 +219,19 @@ public ListenableFuture resolveFleetChips( @Override public ListenableFuture getFleetValueList(FleetValueListRequest request) { - return Futures.submit( - () -> { - Fleet fleet = normalize(request.getFleet()); - return valueLister.listValues( - corpus(fleet, request.getEntity()), request.getKey(), request.getFiltersList()); - }, + Fleet fleet = normalize(request.getFleet()); + Set keys = + isOverlayKey(request.getKey()) ? ImmutableSet.of(request.getKey()) : ImmutableSet.of(); + return Futures.transformAsync( + overlayStore.loadOverlaysAsync(fleet, keys, executor), + overlays -> + Futures.submit( + () -> + valueLister.listValues( + corpus(fleet, request.getEntity(), overlays), + request.getKey(), + request.getFiltersList()), + executor), executor); } @@ -238,22 +257,67 @@ public ListenableFuture getFleetColumnCatalog( executor); } - /** - * Builds the search corpus for a fleet and entity. A host search projects the fleet through a - * {@link HostCorpus} over the host index and host posting lists; every other entity projects it - * through a {@link DeviceCorpus}. A missing curation is passed through as null so the promoted - * keys provider keeps its curation-missing fallback. - */ - private SearchCorpus corpus(Fleet fleet, SearchEntity entity) { + private SearchCorpus corpus( + Fleet fleet, SearchEntity entity, ImmutableMap overlays) { if (entity == SearchEntity.SEARCH_ENTITY_HOST) { return new HostCorpus(store.get(fleet), store.hostPostings(fleet), curations.get(fleet)); } - return new DeviceCorpus(store.get(fleet), store.postings(fleet), curations.get(fleet)); + FleetSnapshot snapshot = store.get(fleet); + OverlayView overlayView = OverlayView.bind(snapshot, overlays); + return new DeviceCorpus(snapshot, store.postings(fleet), curations.get(fleet), overlayView); + } + + private SearchCorpus corpus(Fleet fleet, SearchEntity entity) { + return corpus(fleet, entity, ImmutableMap.of()); + } + + private static Set extractReferencedKeys(FleetSearchRequest request) { + Set keys = new HashSet<>(); + for (Filter filter : request.getFiltersList()) { + if (isOverlayKey(filter.getKey())) { + keys.add(filter.getKey()); + } + } + switch (request.getViewCase()) { + case FLAT -> { + FleetFlatView flat = request.getFlat(); + for (String col : flat.getColumnsList()) { + if (isOverlayKey(col)) { + keys.add(col); + } + } + if (flat.hasSort() && isOverlayKey(flat.getSort().getKey())) { + keys.add(flat.getSort().getKey()); + } + } + case GROUP_HEADER -> { + FleetGroupHeaderView header = request.getGroupHeader(); + for (String gb : header.getGroupByList()) { + if (isOverlayKey(gb)) { + keys.add(gb); + } + } + if (header.hasSort() && isOverlayKey(header.getSort().getField().getGroupKey())) { + keys.add(header.getSort().getField().getGroupKey()); + } + } + case GROUP_EXPAND -> { + FleetGroupExpandView expand = request.getGroupExpand(); + for (String col : expand.getColumnsList()) { + if (isOverlayKey(col)) { + keys.add(col); + } + } + } + case VIEW_NOT_SET -> {} + } + return keys; + } + + private static boolean isOverlayKey(String keyId) { + return keyId != null && keyId.startsWith("dim::"); } - /** - * Normalizes an unspecified fleet to the self fleet, matching the {@code Fleet} proto default. - */ private static Fleet normalize(Fleet fleet) { return fleet == Fleet.FLEET_UNSPECIFIED ? Fleet.FLEET_SELF : fleet; } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD index 4bf8e62b42..9a84ff32be 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD @@ -94,7 +94,10 @@ java_library( java_library( name = "fleet_index", - srcs = ["FleetIndex.java"], + srcs = [ + "CoreFleetIndex.java", + "FleetIndex.java", + ], deps = [ ":key_count", ":value_key_pair", @@ -123,6 +126,12 @@ java_library( ], ) +java_library( + name = "postings", + srcs = ["Postings.java"], + deps = ["@maven//:com_google_guava_guava"], +) + java_library( name = "lazy_postings", srcs = ["LazyPostings.java"], @@ -131,6 +140,7 @@ java_library( ":device_value_extractor", ":host_record", ":host_value_extractor", + ":postings", "@maven//:com_google_guava_guava", ], ) @@ -172,3 +182,46 @@ java_library( "@maven//:javax_inject_jsr330_api", ], ) + +java_library( + name = "dimension_overlay", + srcs = ["DimensionOverlay.java"], + deps = [ + ":fleet_snapshot", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:dimension_overlay_raw", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "overlay_view", + srcs = ["OverlayView.java"], + deps = [ + ":dimension_overlay", + ":fleet_snapshot", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "composite_fleet_index", + srcs = ["CompositeFleetIndex.java"], + deps = [ + ":fleet_index", + ":key_count", + ":overlay_view", + ":value_key_pair", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "composite_postings", + srcs = ["CompositePostings.java"], + deps = [ + ":overlay_view", + ":postings", + "@maven//:com_google_guava_guava", + ], +) diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositeFleetIndex.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositeFleetIndex.java new file mode 100644 index 0000000000..fd91004cfb --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositeFleetIndex.java @@ -0,0 +1,100 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** + * Composite implementation of {@link FleetIndex} combining a core index with an {@link + * OverlayView}. + * + *

Delegates per-key lookups to the core index when the key is present in core, and falls back to + * the overlay view for on-demand long-tail dimensions. + * + *

In accordance with decision D6, global bare-value search indices ({@link + * #semanticGlobalSorted} and {@link #globalExact}) are strictly delegated to the core index only, + * isolating global value search from long-tail dimension noise and ensuring replica consistency. + */ +public final class CompositeFleetIndex implements FleetIndex { + + private final FleetIndex core; + private final OverlayView overlay; + + public CompositeFleetIndex(FleetIndex core, OverlayView overlay) { + this.core = checkNotNull(core); + this.overlay = checkNotNull(overlay); + } + + @Override + public ImmutableList sortedValues(String keyId) { + if (core.keyIds().contains(keyId)) { + return core.sortedValues(keyId); + } + return overlay.sortedValues(keyId); + } + + @Override + public ImmutableMap valueCounts(String keyId) { + if (core.keyIds().contains(keyId)) { + return core.valueCounts(keyId); + } + return overlay.valueCounts(keyId); + } + + @Override + public ImmutableMap valueDisplays(String keyId) { + if (core.keyIds().contains(keyId)) { + return core.valueDisplays(keyId); + } + return overlay.valueDisplays(keyId); + } + + @Override + public String displayName(String keyId) { + return core.displayName(keyId); + } + + @Override + public int valueCount(String keyId, String value) { + if (core.keyIds().contains(keyId)) { + return core.valueCount(keyId, value); + } + return overlay.valueCount(keyId, value); + } + + @Override + public ImmutableSet keyIds() { + return ImmutableSet.builder() + .addAll(core.keyIds()) + .addAll(overlay.loadedKeys()) + .build(); + } + + @Override + public ImmutableList semanticGlobalSorted() { + return core.semanticGlobalSorted(); + } + + @Override + public ImmutableMap> globalExact() { + return core.globalExact(); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositePostings.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositePostings.java new file mode 100644 index 0000000000..3e1ba7740e --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositePostings.java @@ -0,0 +1,54 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.collect.ImmutableMap; + +/** + * Composite implementation of {@link Postings} combining core postings with an {@link OverlayView}. + * + *

Delegates posting list lookups to the core postings when the key is not managed by overlay, + * and resolves on-demand long-tail dimensions through the overlay view. + */ +public final class CompositePostings implements Postings { + + private final Postings core; + private final OverlayView overlay; + + public CompositePostings(Postings core, OverlayView overlay) { + this.core = checkNotNull(core); + this.overlay = checkNotNull(overlay); + } + + @Override + public int[] get(String keyId, String value) { + if (overlay.containsKey(keyId)) { + return overlay.getPostings(keyId, value); + } + return core.get(keyId, value); + } + + @Override + public ImmutableMap forKey(String keyId) { + if (overlay.containsKey(keyId)) { + return overlay.postingsForKey(keyId); + } + return core.forKey(keyId); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CoreFleetIndex.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CoreFleetIndex.java new file mode 100644 index 0000000000..d6f7fece35 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/CoreFleetIndex.java @@ -0,0 +1,126 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** + * Immutable value index over one fleet's core snapshot records. + * + *

Built by {@link FleetIndexBuilder} and held by {@link FleetSnapshot}. Implements the {@link + * FleetIndex} interface over in-memory ImmutableMaps. + */ +@AutoValue +public abstract class CoreFleetIndex implements FleetIndex { + + /** Internal map: key id to (normalized value to device count). */ + public abstract ImmutableMap> valueCountsMap(); + + /** Internal map: key id to its sorted distinct normalized values. */ + public abstract ImmutableMap> sortedValuesMap(); + + /** Internal map: key id to (normalized value to first-seen original display value). */ + public abstract ImmutableMap> valueDisplaysMap(); + + /** Internal map: key id to human-readable display name. */ + public abstract ImmutableMap displayNamesMap(); + + @Override + public ImmutableList sortedValues(String keyId) { + ImmutableList values = sortedValuesMap().get(keyId); + return values != null ? values : ImmutableList.of(); + } + + @Override + public ImmutableMap valueCounts(String keyId) { + ImmutableMap counts = valueCountsMap().get(keyId); + return counts != null ? counts : ImmutableMap.of(); + } + + @Override + public ImmutableMap valueDisplays(String keyId) { + ImmutableMap displays = valueDisplaysMap().get(keyId); + return displays != null ? displays : ImmutableMap.of(); + } + + @Override + public String displayName(String keyId) { + String name = displayNamesMap().get(keyId); + return name != null ? name : FleetIndex.deriveDisplayName(keyId); + } + + @Override + public int valueCount(String keyId, String value) { + ImmutableMap values = valueCountsMap().get(keyId); + return values == null ? 0 : values.getOrDefault(value, 0); + } + + @Override + public abstract ImmutableSet keyIds(); + + @Override + public abstract ImmutableList semanticGlobalSorted(); + + @Override + public abstract ImmutableMap> globalExact(); + + /** Creates a new builder. */ + public static Builder builder() { + return new AutoValue_CoreFleetIndex.Builder(); + } + + /** An empty index, used by an empty {@link FleetSnapshot}. */ + public static CoreFleetIndex empty() { + return builder() + .setValueCountsMap(ImmutableMap.of()) + .setSortedValuesMap(ImmutableMap.of()) + .setValueDisplaysMap(ImmutableMap.of()) + .setDisplayNamesMap(ImmutableMap.of()) + .setKeyIds(ImmutableSet.of()) + .setSemanticGlobalSorted(ImmutableList.of()) + .setGlobalExact(ImmutableMap.of()) + .build(); + } + + /** Builder for {@link CoreFleetIndex}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setValueCountsMap( + ImmutableMap> valueCounts); + + public abstract Builder setSortedValuesMap( + ImmutableMap> sortedValues); + + public abstract Builder setValueDisplaysMap( + ImmutableMap> valueDisplays); + + public abstract Builder setDisplayNamesMap(ImmutableMap displayNames); + + public abstract Builder setKeyIds(ImmutableSet keyIds); + + public abstract Builder setSemanticGlobalSorted( + ImmutableList semanticGlobalSorted); + + public abstract Builder setGlobalExact( + ImmutableMap> globalExact); + + public abstract CoreFleetIndex build(); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlay.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlay.java new file mode 100644 index 0000000000..f4a6ae38cf --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlay.java @@ -0,0 +1,216 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.primitives.Ints; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.DimensionOverlayRaw; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * In-memory index and posting cache for one on-demand long-tail dimension. + * + *

Holds the snapshot-independent ground truth ({@link #uuidToValues}, {@link #valueCounts}, + * {@link #sortedValues}, {@link #valueDisplays}) derived purely from the pulled dimension data. + * + *

Posting lists ({@code value -> int[] deviceIndex}) are bound to a specific snapshot's device + * ordering via {@link #bind(FleetSnapshot)}. When a snapshot rotates, postings are re-indexed in + * <1 ms against {@link FleetSnapshot#uuidToIndex()}. + */ +public final class DimensionOverlay { + + private final String keyId; + private final ImmutableMap> uuidToValues; + private final ImmutableMap valueCounts; + private final ImmutableList sortedValues; + private final ImmutableMap valueDisplays; + + private volatile SnapshotBoundPostings boundPostings; + + private DimensionOverlay( + String keyId, + ImmutableMap> uuidToValues, + ImmutableMap valueCounts, + ImmutableList sortedValues, + ImmutableMap valueDisplays, + SnapshotBoundPostings boundPostings) { + this.keyId = checkNotNull(keyId); + this.uuidToValues = checkNotNull(uuidToValues); + this.valueCounts = checkNotNull(valueCounts); + this.sortedValues = checkNotNull(sortedValues); + this.valueDisplays = checkNotNull(valueDisplays); + this.boundPostings = checkNotNull(boundPostings); + } + + public String keyId() { + return keyId; + } + + public ImmutableMap> uuidToValues() { + return uuidToValues; + } + + public ImmutableMap valueCounts() { + return valueCounts; + } + + public ImmutableList sortedValues() { + return sortedValues; + } + + public ImmutableMap valueDisplays() { + return valueDisplays; + } + + public int valueCount(String value) { + return valueCounts.getOrDefault(value, 0); + } + + public ImmutableList valuesForUuid(String uuid) { + ImmutableList values = uuidToValues.get(uuid); + return values != null ? values : ImmutableList.of(); + } + + public ImmutableSet lowerValuesForUuid(String uuid) { + ImmutableList values = uuidToValues.get(uuid); + if (values == null || values.isEmpty()) { + return ImmutableSet.of(); + } + return values.stream().map(Ascii::toLowerCase).collect(toImmutableSet()); + } + + /** + * Returns posting lists aligned with the given snapshot. Re-indexes in <1 ms if the snapshot + * has rotated since the last bind. + */ + @CanIgnoreReturnValue + public SnapshotBoundPostings bind(FleetSnapshot snapshot) { + SnapshotBoundPostings current = boundPostings; + if (current.buildTime().equals(snapshot.buildTime())) { + return current; + } + SnapshotBoundPostings updated = reindex(snapshot); + this.boundPostings = updated; + return updated; + } + + private SnapshotBoundPostings reindex(FleetSnapshot snapshot) { + ImmutableMap uuidToIndex = snapshot.uuidToIndex(); + Map> postingsMap = new HashMap<>(); + + for (Map.Entry> entry : uuidToValues.entrySet()) { + String uuid = entry.getKey(); + Integer deviceIndex = uuidToIndex.get(uuid); + if (deviceIndex == null) { + continue; + } + Set seenValues = new HashSet<>(); + for (String rawVal : entry.getValue()) { + String valLower = Ascii.toLowerCase(rawVal); + if (!valLower.isEmpty() && seenValues.add(valLower)) { + postingsMap.computeIfAbsent(valLower, k -> new ArrayList<>()).add(deviceIndex); + } + } + } + + ImmutableMap.Builder frozen = ImmutableMap.builder(); + for (Map.Entry> entry : postingsMap.entrySet()) { + List list = entry.getValue(); + Collections.sort(list); + frozen.put(entry.getKey(), Ints.toArray(list)); + } + return new SnapshotBoundPostings(snapshot.buildTime(), frozen.buildOrThrow()); + } + + /** Builds a {@link DimensionOverlay} from raw pulled data and initial snapshot. */ + public static DimensionOverlay create(DimensionOverlayRaw raw, FleetSnapshot snapshot) { + String keyId = raw.keyId(); + ImmutableMap> uuidToValues = raw.uuidToValues(); + + Map valueCounts = new HashMap<>(); + Map valueDisplays = new HashMap<>(); + Set distinctValues = new HashSet<>(); + + for (ImmutableList values : uuidToValues.values()) { + Set seenForDevice = new HashSet<>(); + for (String rawVal : values) { + String valLower = Ascii.toLowerCase(rawVal); + if (!valLower.isEmpty() && seenForDevice.add(valLower)) { + distinctValues.add(valLower); + valueCounts.merge(valLower, 1, Integer::sum); + valueDisplays.putIfAbsent(valLower, rawVal); + } + } + } + + List sortedList = new ArrayList<>(distinctValues); + Collections.sort(sortedList); + + DimensionOverlay overlay = + new DimensionOverlay( + keyId, + uuidToValues, + ImmutableMap.copyOf(valueCounts), + ImmutableList.copyOf(sortedList), + ImmutableMap.copyOf(valueDisplays), + new SnapshotBoundPostings(Instant.EPOCH, ImmutableMap.of())); + + // Bind to the given snapshot immediately. + overlay.bind(snapshot); + return overlay; + } + + /** Posting lists bound to a specific snapshot build time. */ + public static final class SnapshotBoundPostings { + private final Instant buildTime; + private final ImmutableMap postings; + + SnapshotBoundPostings(Instant buildTime, ImmutableMap postings) { + this.buildTime = buildTime; + this.postings = postings; + } + + public Instant buildTime() { + return buildTime; + } + + public int[] get(String valueLower) { + int[] posting = postings.get(valueLower); + return posting != null ? posting : EMPTY_POSTING; + } + + public ImmutableMap postings() { + return postings; + } + + private static final int[] EMPTY_POSTING = new int[0]; + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndex.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndex.java index db552c63d2..e6d524414c 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndex.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndex.java @@ -16,108 +16,70 @@ package com.google.devtools.mobileharness.fe.v6.service.search.index; -import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; /** - * Value index over one fleet's devices. + * Value index interface over a fleet's devices or hosts. * - *

Built by {@link FleetIndexBuilder} from a {@code FleetRawData} and held by {@link - * FleetSnapshot}. Serves value resolution and facet counts from memory. Posting lists (device index - * arrays per key-value pair) are built lazily by {@link LazyPostings} on first access rather than - * at index build time, which keeps the build under 2 seconds for 152K devices. + *

Serves value resolution, counts, sorted values, display names, and facet counts from memory. + * Keys are identified by a namespaced key id (e.g. {@code field::status}, {@code dim::model}, + * {@code prop::rack_id}, {@code host::host_name}, {@code config::wifi_ssid}). * - *

All structures key values by a normalized (lowercased) form so lookups are case-insensitive; - * {@link #valueDisplays()} keeps the original casing for presentation. - * - *

Keys are identified by a namespaced key id, matching the search prototype: {@code - * field::} for built-in device fields (uuid, status, owner, type, driver, decorator, - * executor), {@code dim::} for composite dimensions, {@code prop::} for host - * properties, {@code host::} for cross-entity host attributes joined onto each device, and - * {@code config::} for config-service fields such as wifi_ssid. + *

Values are lowercased for case-insensitive lookup. */ -@AutoValue -public abstract class FleetIndex { +public interface FleetIndex { + + /** Returns the sorted distinct normalized values for a key, or an empty list if absent. */ + ImmutableList sortedValues(String keyId); /** - * Key id to (normalized value to device count). The count is a distinct-device count: a device - * that lists the same value twice contributes one. + * Returns the (normalized value -> distinct-device count) map for a key, or an empty map if + * absent. */ - public abstract ImmutableMap> valueCounts(); + ImmutableMap valueCounts(String keyId); - /** Key id to its sorted distinct normalized values. Used for prefix matching. */ - public abstract ImmutableMap> sortedValues(); + /** + * Returns the (normalized value -> original display value) map for a key, or an empty map if + * absent. + */ + ImmutableMap valueDisplays(String keyId); - /** Key id to (normalized value to first-seen original display value). */ - public abstract ImmutableMap> valueDisplays(); + /** Returns the human-readable display name for a key, deriving it from namespace if absent. */ + String displayName(String keyId); - /** All key ids present in this fleet. */ - public abstract ImmutableSet keyIds(); + /** Returns the device/record count for a value, or 0 if the key or value is absent. */ + int valueCount(String keyId, String value); - /** Key id to human-readable display name. */ - public abstract ImmutableMap displayNames(); + /** Returns all key ids available in this index. */ + ImmutableSet keyIds(); /** - * All (value, key) pairs from non-{@link FleetSearchKeys#PLAIN_VALUE_KEYS} keys, sorted by value - * then key. The suggestion engine bisects into this list for O(log D_s) prefix matching across - * all semantic keys simultaneously. + * All (value, key) pairs from non-{@link FleetSearchKeys#PLAIN_VALUE_KEYS} semantic keys, sorted + * by value then key. Used for O(log D_s) global value prefix matching in Pattern 4. */ - public abstract ImmutableList semanticGlobalSorted(); + ImmutableList semanticGlobalSorted(); /** - * Normalized value to the list of (key, count) pairs that carry that value. Covers all keys (not - * just semantic), enabling O(1) exact-match lookup for the suggestion engine. + * Normalized value to the list of (key, count) pairs that carry that value. Used for O(1) exact + * global value lookup in Pattern 4. */ - public abstract ImmutableMap> globalExact(); - - /** Returns the device count for a value, or 0 if the key or value is absent. */ - public int valueCount(String keyId, String value) { - ImmutableMap values = valueCounts().get(keyId); - return values == null ? 0 : values.getOrDefault(value, 0); - } - - /** Creates a new builder. */ - public static Builder builder() { - return new AutoValue_FleetIndex.Builder(); - } - - /** An empty index, used by an empty {@link FleetSnapshot}. */ - public static FleetIndex empty() { - return builder() - .setValueCounts(ImmutableMap.of()) - .setSortedValues(ImmutableMap.of()) - .setValueDisplays(ImmutableMap.of()) - .setKeyIds(ImmutableSet.of()) - .setDisplayNames(ImmutableMap.of()) - .setSemanticGlobalSorted(ImmutableList.of()) - .setGlobalExact(ImmutableMap.of()) - .build(); - } + ImmutableMap> globalExact(); - /** Builder for {@link FleetIndex}. */ - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setValueCounts( - ImmutableMap> valueCounts); - - public abstract Builder setSortedValues( - ImmutableMap> sortedValues); - - public abstract Builder setValueDisplays( - ImmutableMap> valueDisplays); - - public abstract Builder setKeyIds(ImmutableSet keyIds); - - public abstract Builder setDisplayNames(ImmutableMap displayNames); - - public abstract Builder setSemanticGlobalSorted( - ImmutableList semanticGlobalSorted); - - public abstract Builder setGlobalExact( - ImmutableMap> globalExact); - - public abstract FleetIndex build(); + /** + * Derives a display name from a key id for keys absent from the built-in display name registry. + * Mirrors the namespace derivation the index builder applies to discovered dimensions and host + * properties. + */ + static String deriveDisplayName(String keyId) { + int separator = keyId.indexOf("::"); + String namespace = separator >= 0 ? keyId.substring(0, separator) : ""; + String name = separator >= 0 ? keyId.substring(separator + 2) : keyId; + return switch (namespace) { + case "dim" -> "Dimension " + name; + case "prop" -> "Host Property " + name; + default -> name; + }; } } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilder.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilder.java index a00344fa1b..604b15d8b3 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilder.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilder.java @@ -262,9 +262,17 @@ record HostDevices(HostRecord host, List devices) {} // Phase 2: flatten + merge T accumulators (not 43K). List allDevices = new ArrayList<>(); ImmutableList.Builder allHosts = ImmutableList.builder(); + ImmutableMap.Builder uuidToIndex = ImmutableMap.builder(); + int devIdx = 0; for (HostDevices hd : hostDevices) { allHosts.add(hd.host()); - allDevices.addAll(hd.devices()); + for (DeviceRecord device : hd.devices()) { + allDevices.add(device); + if (!device.deviceId().isEmpty()) { + uuidToIndex.put(device.deviceId(), devIdx); + } + devIdx++; + } } Accumulator merged = new Accumulator(); @@ -283,6 +291,7 @@ record HostDevices(HostRecord host, List devices) {} .setHosts(allHosts.build()) .setIndex(merged.toIndex()) .setHostIndex(hostMerged.toIndex()) + .setUuidToIndex(uuidToIndex.buildKeepingLast()) .build(); } @@ -718,12 +727,12 @@ FleetIndex toIndex() { names.put(keyId, displayName(keyId)); } - return FleetIndex.builder() - .setValueCounts(ImmutableMap.copyOf(countsMap)) - .setSortedValues(ImmutableMap.copyOf(sortedMap)) - .setValueDisplays(ImmutableMap.copyOf(displaysMap)) + return CoreFleetIndex.builder() + .setValueCountsMap(ImmutableMap.copyOf(countsMap)) + .setSortedValuesMap(ImmutableMap.copyOf(sortedMap)) + .setValueDisplaysMap(ImmutableMap.copyOf(displaysMap)) .setKeyIds(ImmutableSet.copyOf(keyIds)) - .setDisplayNames(names.buildOrThrow()) + .setDisplayNamesMap(names.buildOrThrow()) .setSemanticGlobalSorted(ImmutableList.copyOf(semanticPairs)) .setGlobalExact(frozenGlobalExact.buildOrThrow()) .build(); diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSearchKeys.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSearchKeys.java index 9db36f831e..25c77211d8 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSearchKeys.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSearchKeys.java @@ -64,6 +64,21 @@ private FleetSearchKeys() {} public static final String DIM_QUARANTINED = "dim::quarantined"; + /** + * The 8 built-in core dimensions pulled eagerly during the full fleet refresh. All other + * dimensions are long-tail and pulled on-demand via {@code DimensionOverlayStore}. + */ + public static final ImmutableSet CORE_DIMENSION_NAMES = + ImmutableSet.of( + "model", + "version", + "sdk_version", + "device_type", + "pool", + "host_group", + "sub_device_type", + "run_target"); + // ---- Built-in host keys ---- public static final String HOST_NAME = "host::host_name"; diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSnapshot.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSnapshot.java index 8e9024bf92..fc75d24da6 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSnapshot.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetSnapshot.java @@ -18,6 +18,7 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.time.Instant; /** @@ -50,6 +51,9 @@ public abstract class FleetSnapshot { /** Inverted index and value index over {@link #hosts()}. */ public abstract FleetIndex hostIndex(); + /** Map from device UUID to integer index in {@link #devices()}, used for overlay alignment. */ + public abstract ImmutableMap uuidToIndex(); + /** Convenience: number of devices in this snapshot. */ public int deviceCount() { return devices().size(); @@ -62,7 +66,7 @@ public int hostCount() { /** Creates a new builder. */ public static Builder builder() { - return new AutoValue_FleetSnapshot.Builder(); + return new AutoValue_FleetSnapshot.Builder().setUuidToIndex(ImmutableMap.of()); } /** An empty snapshot, used as the initial state before the first refresh completes. */ @@ -71,8 +75,9 @@ public static FleetSnapshot empty() { .setBuildTime(Instant.EPOCH) .setDevices(ImmutableList.of()) .setHosts(ImmutableList.of()) - .setIndex(FleetIndex.empty()) - .setHostIndex(FleetIndex.empty()) + .setIndex(CoreFleetIndex.empty()) + .setHostIndex(CoreFleetIndex.empty()) + .setUuidToIndex(ImmutableMap.of()) .build(); } @@ -89,6 +94,8 @@ public abstract static class Builder { public abstract Builder setHostIndex(FleetIndex hostIndex); + public abstract Builder setUuidToIndex(ImmutableMap uuidToIndex); + public abstract FleetSnapshot build(); } } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/LazyPostings.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/LazyPostings.java index dffc381a54..2efeb92474 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/LazyPostings.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/LazyPostings.java @@ -31,17 +31,25 @@ * *

Built once per {@link FleetSnapshot} lifetime and discarded when the snapshot is replaced by * the refresh cycle. A key's posting lists are constructed on first access by scanning the forward - * store (O(N) per key, roughly 1 to 2 ms for 152K devices) and cached for subsequent lookups. + * store (O(N) per key: roughly 1 to 2 ms for 152K devices, and less than 1 ms for 43K hosts) and + * cached for subsequent lookups. * *

The forward store is supplied as a record-agnostic {@link RecordValues}, so the same posting - * machinery serves the device index (backed by {@link DeviceValueExtractor}) and the host index - * (backed by {@link HostValueExtractor}). The scan visits records in ascending index order, so a - * posting list is always sorted, and the record index is stable across lookups. + * machinery serves both: + * + *

    + *
  • Device search (via {@link #LazyPostings(ImmutableList)}, backed by {@link + * DeviceValueExtractor}) + *
  • Host search (via {@link #forHosts(ImmutableList)}, backed by {@link HostValueExtractor}) + *
+ * + * The scan visits records in ascending index order, so a posting list is always sorted, and the + * record index is stable across lookups. * *

Thread safety is provided by {@link ConcurrentHashMap#computeIfAbsent}, which guarantees that * at most one thread builds the posting lists for a given key. */ -public final class LazyPostings { +public final class LazyPostings implements Postings { private static final int[] EMPTY = new int[0]; @@ -98,6 +106,7 @@ public ImmutableSet valuesForKey(int index, String keyId) { } /** Returns the posting list for (key, value), or an empty array if absent. */ + @Override public int[] get(String keyId, String value) { ImmutableMap keyPostings = forKey(keyId); int[] posting = keyPostings.get(value); @@ -105,6 +114,7 @@ public int[] get(String keyId, String value) { } /** Builds or returns cached posting lists for all values of a key. */ + @Override public ImmutableMap forKey(String keyId) { return cache.computeIfAbsent(keyId, this::buildKeyPostings); } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/OverlayView.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/OverlayView.java new file mode 100644 index 0000000000..e62b9bc41b --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/OverlayView.java @@ -0,0 +1,208 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** Read view over loaded on-demand dimension overlays for one query/corpus context. */ +public interface OverlayView { + + /** Returns true if the key is present in this overlay view. */ + boolean containsKey(String keyId); + + /** All key ids loaded in this overlay view. */ + ImmutableSet loadedKeys(); + + /** Sorted distinct values for an overlay key, or empty if absent. */ + ImmutableList sortedValues(String keyId); + + /** Value counts for an overlay key, or empty if absent. */ + ImmutableMap valueCounts(String keyId); + + /** Value displays for an overlay key, or empty if absent. */ + ImmutableMap valueDisplays(String keyId); + + /** Value count for a specific (key, value), or 0 if absent. */ + int valueCount(String keyId, String value); + + /** Posting list for a specific (key, value), or empty array if absent. */ + int[] getPostings(String keyId, String value); + + /** All posting lists for an overlay key, or empty map if absent. */ + ImmutableMap postingsForKey(String keyId); + + /** Lowercased values for a specific device index and overlay key. */ + ImmutableSet valuesForKey(int deviceIndex, String keyId); + + /** Original display values for a specific device index and overlay key. */ + ImmutableList displayValues(int deviceIndex, String keyId); + + /** Empty overlay view, used when no overlays are loaded. */ + static OverlayView empty() { + return EmptyOverlayView.INSTANCE; + } + + /** Creates an overlay view binding the given overlays to the specified snapshot. */ + static OverlayView bind(FleetSnapshot snapshot, ImmutableMap overlays) { + if (overlays.isEmpty()) { + return empty(); + } + return new BoundOverlayView(snapshot, overlays); + } + + /** Bound implementation of {@link OverlayView} over a specific snapshot. */ + final class BoundOverlayView implements OverlayView { + private final FleetSnapshot snapshot; + private final ImmutableMap overlays; + + BoundOverlayView(FleetSnapshot snapshot, ImmutableMap overlays) { + this.snapshot = checkNotNull(snapshot); + this.overlays = checkNotNull(overlays); + } + + @Override + public boolean containsKey(String keyId) { + return overlays.containsKey(keyId); + } + + @Override + public ImmutableSet loadedKeys() { + return overlays.keySet(); + } + + @Override + public ImmutableList sortedValues(String keyId) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.sortedValues() : ImmutableList.of(); + } + + @Override + public ImmutableMap valueCounts(String keyId) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.valueCounts() : ImmutableMap.of(); + } + + @Override + public ImmutableMap valueDisplays(String keyId) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.valueDisplays() : ImmutableMap.of(); + } + + @Override + public int valueCount(String keyId, String value) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.valueCount(value) : 0; + } + + @Override + public int[] getPostings(String keyId, String value) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.bind(snapshot).get(value) : EMPTY_INT_ARRAY; + } + + @Override + public ImmutableMap postingsForKey(String keyId) { + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.bind(snapshot).postings() : ImmutableMap.of(); + } + + @Override + public ImmutableSet valuesForKey(int deviceIndex, String keyId) { + if (deviceIndex < 0 || deviceIndex >= snapshot.devices().size()) { + return ImmutableSet.of(); + } + String uuid = snapshot.devices().get(deviceIndex).deviceId(); + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.lowerValuesForUuid(uuid) : ImmutableSet.of(); + } + + @Override + public ImmutableList displayValues(int deviceIndex, String keyId) { + if (deviceIndex < 0 || deviceIndex >= snapshot.devices().size()) { + return ImmutableList.of(); + } + String uuid = snapshot.devices().get(deviceIndex).deviceId(); + DimensionOverlay overlay = overlays.get(keyId); + return overlay != null ? overlay.valuesForUuid(uuid) : ImmutableList.of(); + } + + private static final int[] EMPTY_INT_ARRAY = new int[0]; + } + + /** Default empty implementation of {@link OverlayView}. */ + final class EmptyOverlayView implements OverlayView { + static final EmptyOverlayView INSTANCE = new EmptyOverlayView(); + + private static final int[] EMPTY_INT_ARRAY = new int[0]; + + private EmptyOverlayView() {} + + @Override + public boolean containsKey(String keyId) { + return false; + } + + @Override + public ImmutableSet loadedKeys() { + return ImmutableSet.of(); + } + + @Override + public ImmutableList sortedValues(String keyId) { + return ImmutableList.of(); + } + + @Override + public ImmutableMap valueCounts(String keyId) { + return ImmutableMap.of(); + } + + @Override + public ImmutableMap valueDisplays(String keyId) { + return ImmutableMap.of(); + } + + @Override + public int valueCount(String keyId, String value) { + return 0; + } + + @Override + public int[] getPostings(String keyId, String value) { + return EMPTY_INT_ARRAY; + } + + @Override + public ImmutableMap postingsForKey(String keyId) { + return ImmutableMap.of(); + } + + @Override + public ImmutableSet valuesForKey(int deviceIndex, String keyId) { + return ImmutableSet.of(); + } + + @Override + public ImmutableList displayValues(int deviceIndex, String keyId) { + return ImmutableList.of(); + } + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/Postings.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/Postings.java new file mode 100644 index 0000000000..6e270a603e --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/index/Postings.java @@ -0,0 +1,35 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import com.google.common.collect.ImmutableMap; + +/** + * Inverted index provider for fleet search: maps (keyId, value) pairs to record index arrays. + * + *

Posting lists hold record indices into the corpus (devices or hosts) in ascending order. + * Implementations provide lazy on-demand index construction over the forward store ({@link + * LazyPostings}) or composite overlay views. + */ +public interface Postings { + + /** Returns the posting list for (keyId, value), or an empty array if absent. */ + int[] get(String keyId, String value); + + /** Builds or returns cached posting lists for all values of a key. */ + ImmutableMap forKey(String keyId); +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/AtsOneFleetDataSource.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/AtsOneFleetDataSource.java index 759062012e..cf446d59e2 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/AtsOneFleetDataSource.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/AtsOneFleetDataSource.java @@ -84,6 +84,11 @@ public ListenableFuture pull() { executor); } + @Override + public ListenableFuture pullDimension(String keyId) { + return labInfoFleetPuller.pullDimension(keyId); + } + /** Enumerates the device ids in the lab query result, in lab then device order. */ private static ImmutableList deviceIds(LabQueryResult labData) { if (!labData.hasLabView()) { diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/BUILD index d94dfba4f7..ca06c38960 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/BUILD +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/BUILD @@ -20,16 +20,29 @@ package( default_visibility = ["//src/devtools/mobileharness/fe/v6:visibility"], ) +java_library( + name = "dimension_overlay_raw", + srcs = ["DimensionOverlayRaw.java"], + deps = [ + "//src/java/com/google/devtools/mobileharness/shared/util/auto:auto_value", + "@maven//:com_google_guava_guava", + ], +) + java_library( name = "lab_info_fleet_puller", srcs = ["LabInfoFleetPuller.java"], deps = [ + ":dimension_overlay_raw", + "//src/devtools/mobileharness/api/model/proto:device_java_proto", "//src/devtools/mobileharness/api/query/proto:lab_query_java_proto", "//src/devtools/mobileharness/shared/labinfo/proto:lab_info_service_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_search_keys", "//src/java/com/google/devtools/mobileharness/fe/v6/service/shared/providers:lab_info_provider", "//src/java/com/google/devtools/mobileharness/fe/v6/service/util", "//src/java/com/google/devtools/mobileharness/shared/util/logging:google_logger", "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", "@maven//:javax_inject_jsr330_api", ], ) @@ -38,6 +51,7 @@ java_library( name = "fleet_data_source", srcs = ["FleetDataSource.java"], deps = [ + ":dimension_overlay_raw", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_raw_data", "@maven//:com_google_guava_guava", @@ -48,6 +62,7 @@ java_library( name = "ats_one_fleet_data_source", srcs = ["AtsOneFleetDataSource.java"], deps = [ + ":dimension_overlay_raw", ":fleet_data_source", ":lab_info_fleet_puller", "//src/devtools/mobileharness/api/deviceconfig/proto:device_java_proto", diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/DimensionOverlayRaw.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/DimensionOverlayRaw.java new file mode 100644 index 0000000000..092da49365 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/DimensionOverlayRaw.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.pull; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +/** Raw dimension data pulled from a data source for one on-demand long-tail dimension. */ +@AutoValue +public abstract class DimensionOverlayRaw { + + /** The namespaced key id (e.g. "dim::carrier"). */ + public abstract String keyId(); + + /** Map from device UUID to the list of display-cased values for this dimension. */ + public abstract ImmutableMap> uuidToValues(); + + public static DimensionOverlayRaw create( + String keyId, ImmutableMap> uuidToValues) { + return new AutoValue_DimensionOverlayRaw(keyId, uuidToValues); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/FleetDataSource.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/FleetDataSource.java index 0b892010ed..ecf104869f 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/FleetDataSource.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/FleetDataSource.java @@ -46,4 +46,10 @@ public interface FleetDataSource { * keeps the previously published snapshot for this fleet. */ ListenableFuture pull(); + + /** + * Starts an on-demand pull of a single dimension's values for this fleet and returns a future for + * the raw overlay data. + */ + ListenableFuture pullDimension(String keyId); } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/LabInfoFleetPuller.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/LabInfoFleetPuller.java index 3c0aa03b23..588e7607db 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/LabInfoFleetPuller.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull/LabInfoFleetPuller.java @@ -19,19 +19,28 @@ import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.flogger.FluentLogger; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.devtools.mobileharness.api.model.proto.Device.DeviceDimension; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.DeviceInfo; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabData; import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQuery; import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQueryResult; import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.Page; +import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSearchKeys; import com.google.devtools.mobileharness.fe.v6.service.shared.providers.LabInfoProvider; import com.google.devtools.mobileharness.fe.v6.service.util.UniverseScope; import com.google.devtools.mobileharness.shared.labinfo.proto.LabInfoServiceProto.GetLabInfoRequest; +import com.google.protobuf.FieldMask; +import java.util.LinkedHashSet; +import java.util.Set; import javax.inject.Inject; /** - * Pulls the full fleet from {@code LabInfoService} for the search index refresh cycle. + * Pulls the full fleet or on-demand single dimensions from {@code LabInfoService}. * *

Unlike the per-entity detail-page reads, this asks for the whole fleet in one call: no filter * (every lab and device) and no page limit. It accepts the master's cached data, which is enough @@ -47,17 +56,43 @@ public final class LabInfoFleetPuller { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - /** - * The full fleet request: lab view, no filter, no page limit. Reused across pulls since it never - * varies. - */ + private static final FieldMask FULL_FLEET_FIELD_MASK = + FieldMask.newBuilder() + .addPaths("device_locator.id") + .addPaths("device_status") + .addPaths("device_feature.type") + .addPaths("device_feature.owner") + .addPaths("device_feature.composite_dimension") + .build(); + + /** The full fleet request: lab view, masked core fields, 8 core dimensions, no page limit. */ private static final GetLabInfoRequest FULL_FLEET_REQUEST = GetLabInfoRequest.newBuilder() .setLabQuery( - LabQuery.newBuilder().setLabViewRequest(LabQuery.LabViewRequest.getDefaultInstance())) + LabQuery.newBuilder() + .setLabViewRequest(LabQuery.LabViewRequest.getDefaultInstance()) + .setMask( + LabQuery.Mask.newBuilder() + .setDeviceInfoMask( + LabQuery.Mask.DeviceInfoMask.newBuilder() + .setFieldMask(FULL_FLEET_FIELD_MASK) + .setSupportedDimensionsMask( + LabQuery.Mask.DeviceInfoMask.DimensionsMask.newBuilder() + .addAllDimensionNames( + FleetSearchKeys.CORE_DIMENSION_NAMES)) + .setRequiredDimensionsMask( + LabQuery.Mask.DeviceInfoMask.DimensionsMask.newBuilder() + .addAllDimensionNames( + FleetSearchKeys.CORE_DIMENSION_NAMES))))) .setPage(Page.newBuilder().setLimit(0)) .build(); + private static final FieldMask SINGLE_DIM_FIELD_MASK = + FieldMask.newBuilder() + .addPaths("device_locator.id") + .addPaths("device_feature.composite_dimension") + .build(); + private final LabInfoProvider labInfoProvider; @Inject @@ -70,12 +105,18 @@ public final class LabInfoFleetPuller { * LabQueryResult}. */ public ListenableFuture pull() { + return pull(UniverseScope.SELF); + } + + /** Starts a full-fleet pull for the specified universe. */ + public ListenableFuture pull(UniverseScope universeScope) { logger.atInfo().log( - "Issuing GetLabInfo to the master (SELF universe): full fleet, no page limit, cached" - + " data."); + "Issuing GetLabInfo to the master (%s universe): full fleet, no page limit, cached" + + " data.", + universeScope); Stopwatch stopwatch = Stopwatch.createStarted(); return Futures.transform( - labInfoProvider.getLabInfoAsync(FULL_FLEET_REQUEST, UniverseScope.SELF), + labInfoProvider.getLabInfoAsync(FULL_FLEET_REQUEST, universeScope), response -> { LabQueryResult result = response.getLabQueryResult(); logger.atInfo().log( @@ -85,4 +126,81 @@ public ListenableFuture pull() { }, directExecutor()); } + + /** Starts an on-demand single dimension pull from the specified universe. */ + public ListenableFuture pullDimension( + String keyId, UniverseScope universeScope) { + String dimName = keyId.startsWith("dim::") ? keyId.substring("dim::".length()) : keyId; + + GetLabInfoRequest request = + GetLabInfoRequest.newBuilder() + .setLabQuery( + LabQuery.newBuilder() + .setLabViewRequest(LabQuery.LabViewRequest.getDefaultInstance()) + .setMask( + LabQuery.Mask.newBuilder() + .setDeviceInfoMask( + LabQuery.Mask.DeviceInfoMask.newBuilder() + .setFieldMask(SINGLE_DIM_FIELD_MASK) + .setSupportedDimensionsMask( + LabQuery.Mask.DeviceInfoMask.DimensionsMask.newBuilder() + .addDimensionNames(dimName)) + .setRequiredDimensionsMask( + LabQuery.Mask.DeviceInfoMask.DimensionsMask.newBuilder() + .addDimensionNames(dimName))))) + .setPage(Page.newBuilder().setLimit(0)) + .build(); + + Stopwatch stopwatch = Stopwatch.createStarted(); + return Futures.transform( + labInfoProvider.getLabInfoAsync(request, universeScope), + response -> { + LabQueryResult result = response.getLabQueryResult(); + ImmutableMap.Builder> uuidToValues = ImmutableMap.builder(); + + if (result.hasLabView()) { + for (LabData labData : result.getLabView().getLabDataList()) { + for (DeviceInfo deviceInfo : labData.getDeviceList().getDeviceInfoList()) { + String uuid = deviceInfo.getDeviceLocator().getId(); + if (uuid.isEmpty()) { + continue; + } + Set values = new LinkedHashSet<>(); + for (DeviceDimension dim : + deviceInfo + .getDeviceFeature() + .getCompositeDimension() + .getSupportedDimensionList()) { + if (dim.getName().equals(dimName) && !dim.getValue().isEmpty()) { + values.add(dim.getValue()); + } + } + for (DeviceDimension dim : + deviceInfo + .getDeviceFeature() + .getCompositeDimension() + .getRequiredDimensionList()) { + if (dim.getName().equals(dimName) && !dim.getValue().isEmpty()) { + values.add(dim.getValue()); + } + } + if (!values.isEmpty()) { + uuidToValues.put(uuid, ImmutableList.copyOf(values)); + } + } + } + } + + logger.atInfo().log( + "GetLabInfo on-demand dim '%s' returned %d devices in %d ms.", + keyId, uuidToValues.buildKeepingLast().size(), stopwatch.elapsed().toMillis()); + return DimensionOverlayRaw.create(keyId, uuidToValues.buildKeepingLast()); + }, + directExecutor()); + } + + /** Starts an on-demand single dimension pull from the self universe. */ + public ListenableFuture pullDimension(String keyId) { + return pullDimension(keyId, UniverseScope.SELF); + } } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/BUILD index 7e592e4a91..935db8a89b 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/BUILD +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/BUILD @@ -28,7 +28,7 @@ java_library( "//src/devtools/mobileharness/fe/v6/service/proto/search:search_common_java_proto", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", - "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:lazy_postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:postings", "@maven//:com_google_code_findbugs_jsr305", "@maven//:com_google_guava_guava", ], @@ -43,12 +43,15 @@ java_library( ":search_corpus", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_common_java_proto", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:composite_fleet_index", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:composite_postings", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:device_record", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:device_value_extractor", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_search_keys", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", - "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:lazy_postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:overlay_view", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:postings", "@maven//:com_google_code_findbugs_jsr305", "@maven//:com_google_guava_guava", ], @@ -67,7 +70,7 @@ java_library( "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_search_keys", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:host_value_extractor", - "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:lazy_postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:postings", "@maven//:com_google_code_findbugs_jsr305", "@maven//:com_google_guava_guava", ], @@ -80,7 +83,7 @@ java_library( ":search_corpus", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", - "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:lazy_postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:postings", "@maven//:com_google_guava_guava", "@maven//:javax_inject_jsr330_api", ], @@ -133,7 +136,7 @@ java_library( ":search_corpus", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", - "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:lazy_postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:postings", "@maven//:com_google_guava_guava", "@maven//:javax_inject_jsr330_api", ], @@ -222,7 +225,7 @@ java_library( "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_search_keys", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:key_count", - "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:lazy_postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:postings", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:value_key_pair", "@maven//:com_google_code_findbugs_jsr305", "@maven//:com_google_guava_guava", @@ -238,7 +241,7 @@ java_library( "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_search_keys", - "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:lazy_postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:postings", "@maven//:com_google_guava_guava", "@maven//:javax_inject_jsr330_api", ], diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/DeviceCorpus.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/DeviceCorpus.java index 90bd3c01c2..c18ad3cf54 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/DeviceCorpus.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/DeviceCorpus.java @@ -16,6 +16,8 @@ package com.google.devtools.mobileharness.fe.v6.service.search.query; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -23,12 +25,16 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.Column; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetUtilization; import com.google.devtools.mobileharness.fe.v6.service.proto.search.SearchEntity; +import com.google.devtools.mobileharness.fe.v6.service.proto.search.TextCell; +import com.google.devtools.mobileharness.fe.v6.service.search.index.CompositeFleetIndex; +import com.google.devtools.mobileharness.fe.v6.service.search.index.CompositePostings; import com.google.devtools.mobileharness.fe.v6.service.search.index.DeviceRecord; import com.google.devtools.mobileharness.fe.v6.service.search.index.DeviceValueExtractor; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndex; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSearchKeys; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; -import com.google.devtools.mobileharness.fe.v6.service.search.index.LazyPostings; +import com.google.devtools.mobileharness.fe.v6.service.search.index.OverlayView; +import com.google.devtools.mobileharness.fe.v6.service.search.index.Postings; import java.util.List; import java.util.Optional; import javax.annotation.Nullable; @@ -38,8 +44,7 @@ * *

Records are the snapshot's devices, identified by device UUID. The value projection delegates * to {@link DeviceValueExtractor} (lowercased sets) and {@link FleetCellMapper} (display values, - * headers, typed cells), so it mirrors exactly what the device index builder recorded. Utilization - * is the device idle / busy / other bucketing, so a device group carries a utilization breakdown. + * headers, typed cells), falling back to {@link OverlayView} for on-demand long-tail dimensions. */ public final class DeviceCorpus implements SearchCorpus { @@ -59,24 +64,36 @@ public final class DeviceCorpus implements SearchCorpus { "FASTBOOTDMODE"); private final FleetSnapshot snapshot; - private final LazyPostings postings; + private final FleetIndex index; + private final Postings postings; + private final OverlayView overlayView; @Nullable private final ScenarioCuration curation; private final FleetCellMapper cellMapper = new FleetCellMapper(); public DeviceCorpus( - FleetSnapshot snapshot, LazyPostings postings, @Nullable ScenarioCuration curation) { - this.snapshot = snapshot; - this.postings = postings; + FleetSnapshot snapshot, + Postings postings, + @Nullable ScenarioCuration curation, + OverlayView overlayView) { + this.snapshot = checkNotNull(snapshot); + this.overlayView = checkNotNull(overlayView); + this.index = new CompositeFleetIndex(snapshot.index(), overlayView); + this.postings = new CompositePostings(postings, overlayView); this.curation = curation; } + public DeviceCorpus( + FleetSnapshot snapshot, Postings postings, @Nullable ScenarioCuration curation) { + this(snapshot, postings, curation, OverlayView.empty()); + } + @Override public FleetIndex index() { - return snapshot.index(); + return index; } @Override - public LazyPostings postings() { + public Postings postings() { return postings; } @@ -107,21 +124,34 @@ public boolean plainValueKey(String keyId) { @Override public ImmutableSet valuesForKey(int index, String keyId) { + if (overlayView.containsKey(keyId)) { + return overlayView.valuesForKey(index, keyId); + } return DeviceValueExtractor.valuesForKey(snapshot.devices().get(index), keyId); } @Override public ImmutableList displayValues(int index, String keyId) { + if (overlayView.containsKey(keyId)) { + return overlayView.displayValues(index, keyId); + } return FleetCellMapper.displayValues(snapshot.devices().get(index), keyId, snapshot); } @Override public Column column(String keyId) { - return cellMapper.column(keyId, snapshot); + return Column.newBuilder().setKey(keyId).setDisplayName(index.displayName(keyId)).build(); } @Override public Cell cell(int index, String keyId) { + if (overlayView.containsKey(keyId)) { + return Cell.newBuilder() + .setText( + TextCell.newBuilder() + .setValue(String.join(", ", overlayView.displayValues(index, keyId)))) + .build(); + } return cellMapper.cell(keyId, snapshot.devices().get(index), snapshot); } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetCellMapper.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetCellMapper.java index cbd7e74897..021cc9c5a8 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetCellMapper.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetCellMapper.java @@ -95,7 +95,7 @@ public final class FleetCellMapper { * present in the fleet, and falls back to a name derived from the key namespace otherwise. */ public Column column(String keyId, FleetSnapshot snapshot) { - String display = snapshot.index().displayNames().getOrDefault(keyId, deriveDisplayName(keyId)); + String display = snapshot.index().displayName(keyId); return Column.newBuilder().setKey(keyId).setDisplayName(display).build(); } @@ -209,8 +209,7 @@ private static ImmutableList atsControllerValues( ImmutableList.of( snapshot .index() - .valueDisplays() - .getOrDefault(HOST_ATS_CONTROLLER, ImmutableMap.of()) + .valueDisplays(HOST_ATS_CONTROLLER) .getOrDefault(Ascii.toLowerCase(id), id))) .orElse(ImmutableList.of()); } @@ -231,19 +230,4 @@ private static ImmutableList prefixedValues(DeviceRecord device, String private static ImmutableList singleton(String value) { return value.isEmpty() ? ImmutableList.of() : ImmutableList.of(value); } - - /** - * Derives a display name from a key id for keys absent from the fleet index. Mirrors the - * namespace derivation the index builder applies to discovered dimensions and host properties. - */ - private static String deriveDisplayName(String keyId) { - int separator = keyId.indexOf("::"); - String namespace = separator >= 0 ? keyId.substring(0, separator) : ""; - String name = separator >= 0 ? keyId.substring(separator + 2) : keyId; - return switch (namespace) { - case "dim" -> "Dimension " + name; - case "prop" -> "Host Property " + name; - default -> name; - }; - } } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetChipResolver.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetChipResolver.java index da94a14b05..448c872457 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetChipResolver.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetChipResolver.java @@ -22,7 +22,6 @@ import static com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSearchKeys.VALUE_DISPLAY_KEYS; import com.google.common.base.Ascii; -import com.google.common.collect.ImmutableMap; import com.google.devtools.mobileharness.fe.v6.service.proto.search.ComplexMatch; import com.google.devtools.mobileharness.fe.v6.service.proto.search.ContainsSubstring; import com.google.devtools.mobileharness.fe.v6.service.proto.search.Filter; @@ -45,8 +44,8 @@ * {@code _pill_key}, {@code _pill_condition}, and {@code _bff_metadata} helpers. * *

Resolution is stateless beyond the snapshot: it needs only the chip structure to produce - * display strings. It reads {@link FleetIndex#displayNames} for the human key name and {@link - * FleetIndex#valueDisplays} for original value casing, mirroring the conventions in {@link + * display strings. It reads {@link FleetIndex#displayName} for the human key name and {@link + * FleetIndex#valueDisplays(String)} for original value casing, mirroring the conventions in {@link * FleetCellMapper} and {@link FleetValueLister}. The response arrays are parallel to the request: * {@code filter_chips[i]} resolves {@code filters[i]} and {@code group_by_chips[j]} resolves {@code * group_by_keys[j]}. @@ -128,7 +127,7 @@ private static String pillKey(FleetIndex index, String keyId) { * The full key display name, falling back to a namespace-derived name when absent from the fleet. */ private static String displayName(FleetIndex index, String keyId) { - return index.displayNames().getOrDefault(keyId, deriveDisplayName(keyId)); + return index.displayName(keyId); } private static String conditionText(FleetIndex index, Filter filter) { @@ -228,29 +227,6 @@ private static String setText(FleetIndex index, String keyId, List value * FleetValueLister}'s display resolution. */ private static String displayValue(FleetIndex index, String keyId, String value) { - ImmutableMap displays = index.valueDisplays().get(keyId); - if (displays != null) { - String display = displays.get(Ascii.toLowerCase(value)); - if (display != null) { - return display; - } - } - return value; - } - - /** - * Derives a display name from a key id for keys absent from the fleet index. Mirrors the - * namespace derivation {@link FleetCellMapper} and the index builder apply to discovered - * dimensions and host properties. - */ - private static String deriveDisplayName(String keyId) { - int separator = keyId.indexOf("::"); - String namespace = separator >= 0 ? keyId.substring(0, separator) : ""; - String name = separator >= 0 ? keyId.substring(separator + 2) : keyId; - return switch (namespace) { - case "dim" -> "Dimension " + name; - case "prop" -> "Host Property " + name; - default -> name; - }; + return index.valueDisplays(keyId).getOrDefault(Ascii.toLowerCase(value), value); } } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetColumnCataloger.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetColumnCataloger.java index 4995c017ff..6fc901dab9 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetColumnCataloger.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetColumnCataloger.java @@ -28,7 +28,7 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetColumnCatalogResponse; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetColumnCatalogSection; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndex; -import com.google.devtools.mobileharness.fe.v6.service.search.index.LazyPostings; +import com.google.devtools.mobileharness.fe.v6.service.search.index.Postings; import java.util.ArrayList; import java.util.BitSet; import java.util.Comparator; @@ -121,7 +121,7 @@ public FleetColumnCatalogResponse getColumnCatalog( Comparator byCoverage = Comparator.comparingInt(keyId -> -deviceCounts.getOrDefault(keyId, 0)) - .thenComparing(keyId -> Ascii.toLowerCase(displayName(index, keyId))); + .thenComparing(keyId -> Ascii.toLowerCase(index.displayName(keyId))); // Partition the present keys into the browse buckets, matching the prototype's namespace split: // dim:: keys are dimensions (unless redundant), prop:: keys are host properties, and everything @@ -140,7 +140,7 @@ public FleetColumnCatalogResponse getColumnCatalog( builtin.add(keyId); } } - builtin.sort(Comparator.comparing(keyId -> displayName(index, keyId))); + builtin.sort(Comparator.comparing(index::displayName)); dimensions.sort(byCoverage); properties.sort(byCoverage); @@ -262,7 +262,7 @@ private static void addSearchSection( if (redundant.contains(keyId)) { continue; } - if (norm(displayName(index, keyId)).contains(normalizedQuery) + if (norm(index.displayName(keyId)).contains(normalizedQuery) || norm(bareName(keyId)).contains(normalizedQuery)) { hits.add(keyId); } @@ -332,7 +332,7 @@ private static ImmutableSet redundantDims(SearchCorpus corpus, FleetInde * the prototype's precomputed {@code key_device_count}. */ private static ImmutableMap keyDeviceCounts( - FleetIndex index, LazyPostings postings) { + FleetIndex index, Postings postings) { ImmutableMap.Builder counts = ImmutableMap.builder(); for (String keyId : index.keyIds()) { BitSet devices = new BitSet(); @@ -350,28 +350,12 @@ private static FleetColumnCatalogEntry entry( String keyId, FleetIndex index, ImmutableMap deviceCounts, String reason) { return FleetColumnCatalogEntry.newBuilder() .setKey(keyId) - .setDisplayName(displayName(index, keyId)) + .setDisplayName(index.displayName(keyId)) .setDeviceCount(deviceCounts.getOrDefault(keyId, 0)) .setReason(reason) .build(); } - /** The full key display name, falling back to a namespace-derived name when absent. */ - private static String displayName(FleetIndex index, String keyId) { - return index.displayNames().getOrDefault(keyId, deriveDisplayName(keyId)); - } - - private static String deriveDisplayName(String keyId) { - int separator = keyId.indexOf("::"); - String namespace = separator >= 0 ? keyId.substring(0, separator) : ""; - String name = bareName(keyId); - return switch (namespace) { - case "dim" -> "Dimension " + name; - case "prop" -> "Host Property " + name; - default -> name; - }; - } - /** The bare name of a namespaced key: the segment after the last {@code ::}. */ private static String bareName(String keyId) { int separator = keyId.lastIndexOf("::"); diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetFilterEngine.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetFilterEngine.java index 1e06d7da05..d3933d65bb 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetFilterEngine.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetFilterEngine.java @@ -28,7 +28,7 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.SimpleMatch; import com.google.devtools.mobileharness.fe.v6.service.proto.search.StartsWith; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndex; -import com.google.devtools.mobileharness.fe.v6.service.search.index.LazyPostings; +import com.google.devtools.mobileharness.fe.v6.service.search.index.Postings; import java.util.BitSet; import java.util.HashSet; import java.util.List; @@ -92,7 +92,7 @@ private static BitSet matchFilter(SearchCorpus corpus, Filter filter) { * records that lack the key entirely. When {@code negated} is set the whole result is inverted. */ private static BitSet matchSimple(SearchCorpus corpus, String keyId, SimpleMatch simple) { - LazyPostings postings = corpus.postings(); + Postings postings = corpus.postings(); BitSet include = new BitSet(); for (FilterValue value : simple.getValuesList()) { switch (value.getKindCase()) { @@ -122,8 +122,8 @@ private static BitSet matchComplex(SearchCorpus corpus, String keyId, ComplexMat */ private static BitSet matchStartsWith(SearchCorpus corpus, String keyId, StartsWith startsWith) { FleetIndex index = corpus.index(); - LazyPostings postings = corpus.postings(); - ImmutableList sorted = index.sortedValues().getOrDefault(keyId, ImmutableList.of()); + Postings postings = corpus.postings(); + ImmutableList sorted = index.sortedValues(keyId); String prefix = Ascii.toLowerCase(startsWith.getValue()); int lo = lowerBound(sorted, prefix); // '\uffff' is the largest basic-plane code unit, so prefix + '\uffff' bounds the prefix run. @@ -139,10 +139,10 @@ private static BitSet matchStartsWith(SearchCorpus corpus, String keyId, StartsW private static BitSet matchContains( SearchCorpus corpus, String keyId, ContainsSubstring contains) { FleetIndex index = corpus.index(); - LazyPostings postings = corpus.postings(); + Postings postings = corpus.postings(); String needle = Ascii.toLowerCase(contains.getValue()); BitSet matched = new BitSet(); - for (String value : index.sortedValues().getOrDefault(keyId, ImmutableList.of())) { + for (String value : index.sortedValues(keyId)) { if (value.contains(needle)) { orInto(matched, postings.get(keyId, value)); } @@ -157,7 +157,7 @@ private static BitSet matchContains( */ private static BitSet matchRegex(SearchCorpus corpus, String keyId, MatchesRegex regex) { FleetIndex index = corpus.index(); - LazyPostings postings = corpus.postings(); + Postings postings = corpus.postings(); Pattern pattern; try { pattern = Pattern.compile(regex.getValue(), Pattern.CASE_INSENSITIVE); @@ -165,7 +165,7 @@ private static BitSet matchRegex(SearchCorpus corpus, String keyId, MatchesRegex return negateIfNeeded(new BitSet(), regex.getNegated(), corpus.recordCount()); } BitSet matched = new BitSet(); - for (String value : index.sortedValues().getOrDefault(keyId, ImmutableList.of())) { + for (String value : index.sortedValues(keyId)) { if (pattern.matcher(value).find()) { orInto(matched, postings.get(keyId, value)); } @@ -210,7 +210,7 @@ private static BitSet noValueSet(SearchCorpus corpus, String keyId) { } /** Union of every posting list for the key: the records that carry at least one value for it. */ - private static BitSet recordsWithKey(LazyPostings postings, String keyId) { + private static BitSet recordsWithKey(Postings postings, String keyId) { BitSet withKey = new BitSet(); for (int[] posting : postings.forKey(keyId).values()) { orInto(withKey, posting); @@ -221,8 +221,7 @@ private static BitSet recordsWithKey(LazyPostings postings, String keyId) { /** * Intersection (AND) of the posting lists of the given values. Empty values yield an empty set. */ - private static BitSet intersectPostings( - LazyPostings postings, String keyId, List values) { + private static BitSet intersectPostings(Postings postings, String keyId, List values) { BitSet clause = null; for (String value : values) { BitSet posting = new BitSet(); @@ -276,7 +275,7 @@ private static ImmutableList toSortedList(BitSet set) { /** * Locates the first index in the sorted list whose value is greater than or equal to the key. The - * list must be sorted ascending, matching {@link FleetIndex#sortedValues()}. + * list must be sorted ascending, matching {@link FleetIndex#sortedValues(String)}. */ static int lowerBound(List sorted, String key) { int lo = 0; diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetGroupSearcher.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetGroupSearcher.java index cc4be1acfb..87935a62ec 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetGroupSearcher.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetGroupSearcher.java @@ -315,8 +315,7 @@ private static ImmutableList displayValues( shown.add(NO_VALUE_DISPLAY); continue; } - ImmutableMap displays = - index.valueDisplays().getOrDefault(keys.get(i), ImmutableMap.of()); + ImmutableMap displays = index.valueDisplays(keys.get(i)); List parts = new ArrayList<>(); for (String value : values) { parts.add(displays.getOrDefault(value, value)); diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetPromotedKeysProvider.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetPromotedKeysProvider.java index c882dcd379..a844124239 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetPromotedKeysProvider.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetPromotedKeysProvider.java @@ -242,18 +242,7 @@ private static FleetFilterChipMetadata metadata(FleetIndex index, String keyId) * Mirrors the derivation in {@link FleetChipResolver} and {@link FleetIndexBuilder}. */ private static String displayName(FleetIndex index, String keyId) { - return index.displayNames().getOrDefault(keyId, deriveDisplayName(keyId)); - } - - private static String deriveDisplayName(String keyId) { - int separator = keyId.indexOf("::"); - String namespace = separator >= 0 ? keyId.substring(0, separator) : ""; - String name = separator >= 0 ? keyId.substring(separator + 2) : keyId; - return switch (namespace) { - case "dim" -> "Dimension " + name; - case "prop" -> "Host Property " + name; - default -> name; - }; + return index.displayName(keyId); } /** Distinct value-combination count for a key plus whether some device in the set lacks it. */ diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSearchConfigProvider.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSearchConfigProvider.java index 7f40224aa2..f8b0b46137 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSearchConfigProvider.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSearchConfigProvider.java @@ -116,22 +116,6 @@ private static int browseAllCount(FleetSnapshot snapshot, SearchEntity entity) { * absent. */ private static String displayName(FleetIndex index, String keyId) { - return index.displayNames().getOrDefault(keyId, deriveDisplayName(keyId)); - } - - /** - * Derives a display name from a key id for keys absent from the fleet index. Mirrors the - * namespace derivation in {@code FleetCellMapper} and {@code FleetColumnCataloger}: {@code dim::} - * and {@code prop::} keys are prefixed, and every other namespace shows its bare name. - */ - private static String deriveDisplayName(String keyId) { - int separator = keyId.indexOf("::"); - String namespace = separator >= 0 ? keyId.substring(0, separator) : ""; - String name = separator >= 0 ? keyId.substring(separator + 2) : keyId; - return switch (namespace) { - case "dim" -> "Dimension " + name; - case "prop" -> "Host Property " + name; - default -> name; - }; + return index.displayName(keyId); } } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSuggester.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSuggester.java index dc74f0b4bf..6b363a287a 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSuggester.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetSuggester.java @@ -59,7 +59,7 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.TextSegment; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndex; import com.google.devtools.mobileharness.fe.v6.service.search.index.KeyCount; -import com.google.devtools.mobileharness.fe.v6.service.search.index.LazyPostings; +import com.google.devtools.mobileharness.fe.v6.service.search.index.Postings; import com.google.devtools.mobileharness.fe.v6.service.search.index.ValueKeyPair; import java.util.ArrayList; import java.util.BitSet; @@ -432,12 +432,38 @@ private List suggestKv(Context context, String keyToken, String rawValue, String value = Ascii.toLowerCase(stripQuotes(rawValue.trim())); for (String keyId : keyIds) { + boolean hadMatches = false; for (Match match : matchValues(context.index(), keyId, value, /* allowContains= */ true)) { Cand cand = condition(context, keyId, match.value(), match.tier(), exclude); if (cand != null) { out.add(cand); + hadMatches = true; } } + // Cold long-tail fallback: if the key is valid but has no index entries in core/overlay, + // and is not already in active filters, emit a ready-to-apply filter condition with no count. + if (!hadMatches && !value.isEmpty() && !context.activeKeys().contains(keyId)) { + String display = context.index().displayName(keyId); + String op = exclude ? "is not" : "is"; + ImmutableList mainText = segments(display + " " + op + " ", rawValue.trim()); + Filter filter = + Filter.newBuilder() + .setKey(keyId) + .setSimple( + SimpleMatch.newBuilder() + .addValues(FilterValue.newBuilder().setValue(rawValue.trim())) + .setNegated(exclude)) + .build(); + FleetSuggestion.Builder builder = + FleetSuggestion.newBuilder() + .setLabel("Add filter") + .addAllMainText(mainText) + .setApplyFilter(applyFilter(context.index(), keyId, filter)); + Cand cand = new Cand(Kind.CONDITION, keyId, 1.0, builder, mainTextString(mainText)); + cand.needsCount = false; + cand.noCount = true; + out.add(cand); + } } return out; } @@ -555,8 +581,7 @@ private List suggestValue( // 3. Identifier collapse (PLAIN_VALUE_KEYS, one collapsed suggestion per key). for (String identKey : PLAIN_VALUE_KEYS) { - ImmutableList keyValues = - index.sortedValues().getOrDefault(identKey, ImmutableList.of()); + ImmutableList keyValues = index.sortedValues(identKey); if (keyValues.isEmpty()) { continue; } @@ -978,12 +1003,11 @@ private static int presenceCount(Context context, String keyId) { private static ImmutableList matchValues( FleetIndex index, String keyId, String query, boolean allowContains) { - ImmutableList sorted = index.sortedValues().getOrDefault(keyId, ImmutableList.of()); + ImmutableList sorted = index.sortedValues(keyId); if (sorted.isEmpty() || query.isEmpty()) { return ImmutableList.of(); } - ImmutableMap counts = - index.valueCounts().getOrDefault(keyId, ImmutableMap.of()); + ImmutableMap counts = index.valueCounts(keyId); List out = new ArrayList<>(); Set exactHits = new HashSet<>(); // Full match, normalizing space and underscore both ways (spec section 2.3). @@ -1015,8 +1039,7 @@ private static ImmutableList matchValues( /** Top values of a key by global count, used to offer ready-to-apply conditions. */ private static List topValues(Context context, String keyId, int n) { - ImmutableMap counts = - context.index().valueCounts().getOrDefault(keyId, ImmutableMap.of()); + ImmutableMap counts = context.index().valueCounts(keyId); List all = new ArrayList<>(); for (Map.Entry entry : counts.entrySet()) { if (entry.getValue() > 0) { @@ -1192,18 +1215,7 @@ private static String label(FleetIndex index, String keyId, boolean inChip) { } private static String displayName(FleetIndex index, String keyId) { - return index.displayNames().getOrDefault(keyId, deriveDisplayName(keyId)); - } - - private static String deriveDisplayName(String keyId) { - int separator = keyId.indexOf("::"); - String namespace = separator >= 0 ? keyId.substring(0, separator) : ""; - String name = separator >= 0 ? keyId.substring(separator + 2) : keyId; - return switch (namespace) { - case "dim" -> "Dimension " + name; - case "prop" -> "Host Property " + name; - default -> name; - }; + return index.displayName(keyId); } private static String pillKey(FleetIndex index, String keyId) { @@ -1223,14 +1235,7 @@ private static String bareName(String keyId) { } private static String displayValue(FleetIndex index, String keyId, String valueLower) { - ImmutableMap displays = index.valueDisplays().get(keyId); - if (displays != null) { - String display = displays.get(valueLower); - if (display != null) { - return display; - } - } - return valueLower; + return index.valueDisplays(keyId).getOrDefault(valueLower, valueLower); } /** @@ -1388,7 +1393,7 @@ private static ImmutableList otherFilters(List filters, String k return others.build(); } - private static BitSet devicesWithKey(LazyPostings postings, String keyId) { + private static BitSet devicesWithKey(Postings postings, String keyId) { BitSet withKey = new BitSet(); for (int[] posting : postings.forKey(keyId).values()) { for (int deviceIndex : posting) { @@ -1565,7 +1570,7 @@ private record Context( ImmutableList current, BitSet currentBits, ToIntFunction keyPriority, - LazyPostings postings) {} + Postings postings) {} /** * A candidate suggestion before ranking. Holds the partially built proto (label, main text, and diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetValueLister.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetValueLister.java index d7a9620fab..bbc07b0132 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetValueLister.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/FleetValueLister.java @@ -17,7 +17,6 @@ package com.google.devtools.mobileharness.fe.v6.service.search.query; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.devtools.mobileharness.fe.v6.service.proto.search.Filter; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetCountedNoValueEntry; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetCountedValue; @@ -26,7 +25,7 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetPlainValueList; import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetValueListResponse; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndex; -import com.google.devtools.mobileharness.fe.v6.service.search.index.LazyPostings; +import com.google.devtools.mobileharness.fe.v6.service.search.index.Postings; import java.util.ArrayList; import java.util.BitSet; import java.util.Comparator; @@ -73,9 +72,9 @@ public final class FleetValueLister { public FleetValueListResponse listValues( SearchCorpus corpus, String keyId, List filters) { FleetIndex index = corpus.index(); - LazyPostings postings = corpus.postings(); + Postings postings = corpus.postings(); boolean knownKey = index.keyIds().contains(keyId); - ImmutableList values = index.sortedValues().getOrDefault(keyId, ImmutableList.of()); + ImmutableList values = index.sortedValues(keyId); // The filtered set drops this key's own chip. With no other filters this is the whole fleet, so // every value's filtered count equals its total, which is exactly the prototype's behavior. @@ -96,7 +95,7 @@ private static FleetCountedValueList buildCounted( SearchCorpus corpus, BitSet filteredSet, boolean knownKey, - LazyPostings postings) { + Postings postings) { // Collect in the index's ascending value order so equal filtered counts stay value-ascending // after the stable sort below. List entries = new ArrayList<>(); @@ -152,14 +151,7 @@ private static ImmutableList otherFilters(List filters, String k /** The value's original-casing display, falling back to the normalized value when absent. */ private static String displayFor(FleetIndex index, String keyId, String normalizedValue) { - ImmutableMap displays = index.valueDisplays().get(keyId); - if (displays != null) { - String display = displays.get(normalizedValue); - if (display != null) { - return display; - } - } - return normalizedValue; + return index.valueDisplays(keyId).getOrDefault(normalizedValue, normalizedValue); } /** Number of devices in {@code posting} that are also in the filtered set. */ @@ -174,19 +166,19 @@ private static int intersectionCount(int[] posting, BitSet filteredSet) { } /** Fleet-wide count of devices that lack the key entirely. */ - private static int noValueTotal(SearchCorpus corpus, LazyPostings postings, String keyId) { + private static int noValueTotal(SearchCorpus corpus, Postings postings, String keyId) { return corpus.recordCount() - devicesWithKey(postings, keyId).cardinality(); } /** Count of devices in the filtered set that lack the key entirely. */ - private static int noValueFiltered(LazyPostings postings, String keyId, BitSet filteredSet) { + private static int noValueFiltered(Postings postings, String keyId, BitSet filteredSet) { BitSet lacking = (BitSet) filteredSet.clone(); lacking.andNot(devicesWithKey(postings, keyId)); return lacking.cardinality(); } /** Union of every posting list for the key: the devices that carry at least one value for it. */ - private static BitSet devicesWithKey(LazyPostings postings, String keyId) { + private static BitSet devicesWithKey(Postings postings, String keyId) { BitSet withKey = new BitSet(); for (int[] posting : postings.forKey(keyId).values()) { for (int deviceIndex : posting) { diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/HostCellMapper.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/HostCellMapper.java index 641c044ccd..59a26b3739 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/HostCellMapper.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/HostCellMapper.java @@ -80,8 +80,7 @@ public final class HostCellMapper { * present in the fleet, and falls back to a name derived from the key namespace otherwise. */ public Column column(String keyId, FleetSnapshot snapshot) { - String display = - snapshot.hostIndex().displayNames().getOrDefault(keyId, deriveDisplayName(keyId)); + String display = snapshot.hostIndex().displayName(keyId); return Column.newBuilder().setKey(keyId).setDisplayName(display).build(); } @@ -160,8 +159,7 @@ private static ImmutableList atsControllerValues( ImmutableList.of( snapshot .hostIndex() - .valueDisplays() - .getOrDefault(HOST_ATS_CONTROLLER, ImmutableMap.of()) + .valueDisplays(HOST_ATS_CONTROLLER) .getOrDefault(Ascii.toLowerCase(id), id))) .orElse(ImmutableList.of()); } @@ -177,19 +175,4 @@ private static ImmutableList prefixedValues(HostRecord host, String keyI private static ImmutableList singleton(String value) { return value.isEmpty() ? ImmutableList.of() : ImmutableList.of(value); } - - /** - * Derives a display name from a key id for keys absent from the host index. Mirrors the namespace - * derivation the index builder applies to discovered host properties. - */ - private static String deriveDisplayName(String keyId) { - int separator = keyId.indexOf("::"); - String namespace = separator >= 0 ? keyId.substring(0, separator) : ""; - String name = separator >= 0 ? keyId.substring(separator + 2) : keyId; - return switch (namespace) { - case "dim" -> "Dimension " + name; - case "prop" -> "Host Property " + name; - default -> name; - }; - } } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/HostCorpus.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/HostCorpus.java index 6c319ea051..a1f451345b 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/HostCorpus.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/HostCorpus.java @@ -26,7 +26,7 @@ import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSearchKeys; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; import com.google.devtools.mobileharness.fe.v6.service.search.index.HostValueExtractor; -import com.google.devtools.mobileharness.fe.v6.service.search.index.LazyPostings; +import com.google.devtools.mobileharness.fe.v6.service.search.index.Postings; import java.util.List; import java.util.Optional; import javax.annotation.Nullable; @@ -42,12 +42,12 @@ public final class HostCorpus implements SearchCorpus { private final FleetSnapshot snapshot; - private final LazyPostings postings; + private final Postings postings; @Nullable private final ScenarioCuration curation; private final HostCellMapper cellMapper = new HostCellMapper(); public HostCorpus( - FleetSnapshot snapshot, LazyPostings postings, @Nullable ScenarioCuration curation) { + FleetSnapshot snapshot, Postings postings, @Nullable ScenarioCuration curation) { this.snapshot = snapshot; this.postings = postings; this.curation = curation; @@ -59,7 +59,7 @@ public FleetIndex index() { } @Override - public LazyPostings postings() { + public Postings postings() { return postings; } diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/SearchCorpus.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/SearchCorpus.java index a56b2303f3..fdbd995826 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/SearchCorpus.java +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/query/SearchCorpus.java @@ -23,7 +23,7 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.FleetUtilization; import com.google.devtools.mobileharness.fe.v6.service.proto.search.SearchEntity; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndex; -import com.google.devtools.mobileharness.fe.v6.service.search.index.LazyPostings; +import com.google.devtools.mobileharness.fe.v6.service.search.index.Postings; import java.util.List; import java.util.Optional; import javax.annotation.Nullable; @@ -53,7 +53,7 @@ public interface SearchCorpus { FleetIndex index(); /** The lazily built posting lists over the corpus records. */ - LazyPostings postings(); + Postings postings(); /** Number of records in the corpus. */ int recordCount(); diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD index b785d08052..36fad5a8af 100644 --- a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD @@ -48,6 +48,21 @@ java_library( ], ) +java_library( + name = "dimension_overlay_store", + srcs = ["DimensionOverlayStore.java"], + deps = [ + ":fleet_snapshot_store", + "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:dimension_overlay", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:dimension_overlay_raw", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:fleet_data_source", + "@maven//:com_google_guava_guava", + "@maven//:javax_inject_jsr330_api", + ], +) + java_library( name = "fleet_search_data_module", srcs = ["FleetSearchDataModule.java"], diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStore.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStore.java new file mode 100644 index 0000000000..a0ea54263c --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStore.java @@ -0,0 +1,137 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.refresh; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.Futures.immediateFuture; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import com.google.devtools.mobileharness.fe.v6.service.proto.search.Fleet; +import com.google.devtools.mobileharness.fe.v6.service.search.index.DimensionOverlay; +import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.DimensionOverlayRaw; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.FleetDataSource; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; +import javax.inject.Inject; +import javax.inject.Singleton; + +/** + * Thread-safe store for on-demand {@link DimensionOverlay} instances across fleets. + * + *

Asynchronously loads cold dimensions via {@link FleetDataSource#pullDimension(String)} with + * in-flight deduplication and returns strong-reference maps for query execution. + */ +@Singleton +public final class DimensionOverlayStore { + + private final Map dataSources; + private final FleetSnapshotStore snapshotStore; + private final ConcurrentMap> memoryCaches = + new ConcurrentHashMap<>(); + private final ConcurrentMap>> + inFlight = new ConcurrentHashMap<>(); + + @Inject + DimensionOverlayStore(Map dataSources, FleetSnapshotStore snapshotStore) { + this.dataSources = checkNotNull(dataSources); + this.snapshotStore = checkNotNull(snapshotStore); + } + + /** + * Asynchronously loads all requested overlay keys for the given fleet and returns a strong + * reference map of loaded overlays. Keys that are already cached return immediately. + */ + public ListenableFuture> loadOverlaysAsync( + Fleet fleet, Set keyIds, Executor executor) { + if (keyIds.isEmpty()) { + return immediateFuture(ImmutableMap.of()); + } + + FleetDataSource dataSource = dataSources.get(fleet); + if (dataSource == null) { + return immediateFuture(ImmutableMap.of()); + } + + ConcurrentMap cache = + memoryCaches.computeIfAbsent(fleet, f -> new ConcurrentHashMap<>()); + ConcurrentMap> inFlightMap = + inFlight.computeIfAbsent(fleet, f -> new ConcurrentHashMap<>()); + + List>> futures = new ArrayList<>(); + + for (String keyId : keyIds) { + DimensionOverlay cached = cache.get(keyId); + if (cached != null) { + futures.add(immediateFuture(Map.entry(keyId, cached))); + continue; + } + + ListenableFuture pullFuture = inFlightMap.get(keyId); + if (pullFuture == null) { + SettableFuture settable = SettableFuture.create(); + ListenableFuture existing = inFlightMap.putIfAbsent(keyId, settable); + if (existing != null) { + pullFuture = existing; + } else { + pullFuture = settable; + ListenableFuture rawFuture = dataSource.pullDimension(keyId); + Futures.addCallback( + rawFuture, + new FutureCallback<>() { + @Override + public void onSuccess(DimensionOverlayRaw raw) { + try { + FleetSnapshot snapshot = snapshotStore.get(fleet); + DimensionOverlay overlay = DimensionOverlay.create(raw, snapshot); + cache.put(keyId, overlay); + inFlightMap.remove(keyId); + settable.set(overlay); + } catch (Throwable t) { + inFlightMap.remove(keyId); + settable.setException(t); + } + } + + @Override + public void onFailure(Throwable t) { + inFlightMap.remove(keyId); + settable.setException(t); + } + }, + executor); + } + } + + futures.add(Futures.transform(pullFuture, overlay -> Map.entry(keyId, overlay), executor)); + } + + return Futures.transform( + Futures.allAsList(futures), + entries -> ImmutableMap.builder().putAll(entries).buildOrThrow(), + executor); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsDeviceKeyRegistry.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsDeviceKeyRegistry.java new file mode 100644 index 0000000000..148d4285ac --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsDeviceKeyRegistry.java @@ -0,0 +1,33 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import javax.inject.Inject; + +/** + * Device-search key registry for the standalone ATS (OSS) deployment. + * + *

Composes Group 1 (universal common device keys and projected common host keys) with Group 2 + * (Standalone ATS WiFi SSID). + */ +public final class AtsDeviceKeyRegistry extends DeviceKeyRegistry { + + @Inject + AtsDeviceKeyRegistry() { + super(AtsDeviceKeys.ATS_DEVICE_KEYS); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsDeviceKeys.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsDeviceKeys.java new file mode 100644 index 0000000000..3d34bbcb32 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsDeviceKeys.java @@ -0,0 +1,42 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import com.google.common.collect.ImmutableList; + +/** + * Standalone ATS (OSS) exclusive device key descriptors (Group 2). + * + *

Kept separate from {@link DeviceKeys} so that catalog holds only the universal common keys and + * the shared helper methods; this catalog holds the keys that exist only in the standalone ATS + * deployment. + */ +public final class AtsDeviceKeys { + + /** Fed by ConfigService, so it contributes no {@code GetLabInfo} mask. */ + public static final DeviceKeyDescriptor WIFI_SSID = + DeviceKeyDescriptor.builder() + .setId(DeviceKeys.PREFIX_DEVICE_CONFIG + "wifi_ssid") + .setDisplay(KeyDisplay.of("WiFi SSID")) + .build(); + + /** Group 2: Standalone ATS exclusive device keys. */ + public static final ImmutableList ATS_DEVICE_KEYS = + ImmutableList.of(WIFI_SSID); + + private AtsDeviceKeys() {} +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsHostKeyRegistry.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsHostKeyRegistry.java new file mode 100644 index 0000000000..5f11131be2 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/AtsHostKeyRegistry.java @@ -0,0 +1,33 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import com.google.common.collect.ImmutableList; +import javax.inject.Inject; + +/** + * Host-search key registry for the standalone ATS (OSS) deployment. + * + *

Holds only Group 1 (universal common host keys); standalone ATS adds no extra host keys. + */ +public final class AtsHostKeyRegistry extends HostKeyRegistry { + + @Inject + AtsHostKeyRegistry() { + super(ImmutableList.of()); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/BUILD b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/BUILD new file mode 100644 index 0000000000..443e8fd925 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/BUILD @@ -0,0 +1,109 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_java//java:java_library.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src/devtools/mobileharness/fe/v6:visibility"], +) + +java_library( + name = "device_info_source", + srcs = ["DeviceInfoSource.java"], + deps = [ + "//src/devtools/mobileharness/api/model/proto:device_java_proto", + "//src/devtools/mobileharness/api/query/proto:lab_query_java_proto", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "lab_info_source", + srcs = ["LabInfoSource.java"], + deps = [ + "//src/devtools/mobileharness/api/model/proto:lab_java_proto", + "//src/devtools/mobileharness/api/query/proto:lab_query_java_proto", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "key_descriptor", + srcs = [ + "DeviceKeyDescriptor.java", + "HostKeyDescriptor.java", + "KeyDisplay.java", + ], + deps = [ + ":device_info_source", + ":lab_info_source", + "//src/java/com/google/devtools/mobileharness/shared/util/auto:auto_value", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "device_keys", + srcs = ["DeviceKeys.java"], + deps = [ + ":device_info_source", + ":host_keys", + ":key_descriptor", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "host_keys", + srcs = ["HostKeys.java"], + deps = [ + ":key_descriptor", + ":lab_info_source", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "ats_device_keys", + srcs = ["AtsDeviceKeys.java"], + deps = [ + ":device_keys", + ":key_descriptor", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "key_registry", + srcs = [ + "AtsDeviceKeyRegistry.java", + "AtsHostKeyRegistry.java", + "DeviceKeyRegistry.java", + "HostKeyRegistry.java", + ], + deps = [ + ":ats_device_keys", + ":device_info_source", + ":device_keys", + ":host_keys", + ":key_descriptor", + ":lab_info_source", + "//src/devtools/mobileharness/api/query/proto:lab_query_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:javax_inject_jsr330_api", + ], +) diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceInfoSource.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceInfoSource.java new file mode 100644 index 0000000000..7f757a6646 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceInfoSource.java @@ -0,0 +1,126 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.mobileharness.api.model.proto.Device.DeviceCompositeDimension; +import com.google.devtools.mobileharness.api.model.proto.Device.DeviceDimension; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.DeviceInfo; +import java.util.function.Function; +import java.util.stream.Stream; + +/** + * A single component of a device key's value that lives inside the {@code GetLabInfo} response and + * therefore participates in the {@code DeviceInfoMask}. + * + *

{@code Source} exists for one reason: to eliminate DeviceInfo/LabInfo mask drift. Each source + * declares BOTH its mask contribution ({@link #maskFieldPaths()} / {@link #maskDimensionNames()}) + * AND how to read the value from the proto ({@link #extract}), so the two can never diverge. Data + * that is not in {@code GetLabInfo} (HostInfoService, ConfigService, fan-out provenance) is + * deliberately NOT a source: it has no mask, so it carries no drift risk and is handled by + * enrichment instead. + * + *

{@link #extract} returns the raw values of the masked field. Display transforms and multi-feed + * merges are a separate key-level combiner concern (not modeled here). + */ +public abstract class DeviceInfoSource { + + /** The {@code DeviceInfoMask.field_mask} paths this source requires. */ + public abstract ImmutableList maskFieldPaths(); + + /** The {@code DeviceInfoMask} dimension names this source requires. */ + public abstract ImmutableList maskDimensionNames(); + + /** Reads the raw value(s) of this source from a {@code DeviceInfo}. */ + public abstract ImmutableList extract(DeviceInfo deviceInfo); + + /** + * A typed field on {@code DeviceInfo}, named by its {@code DeviceInfoMask} path (for example + * {@code "device_feature.type"}). The path drives the mask and the getter reads the value; both + * are given together at the call site, so a reader sees at a glance that they agree. Every + * derived mask is checked against the proto descriptor in the registry tests, so a mistyped path + * fails there rather than silently yielding empty values at serving time. + */ + public static DeviceInfoSource field( + String protoPath, Function> getter) { + return new FieldSource(protoPath, getter); + } + + /** A named dimension in {@code device_feature.composite_dimension}. */ + public static DeviceInfoSource dimension(String name) { + return new DimensionSource(name); + } + + private static final class FieldSource extends DeviceInfoSource { + private final String protoPath; + private final Function> getter; + + FieldSource(String protoPath, Function> getter) { + this.protoPath = protoPath; + this.getter = getter; + } + + @Override + public ImmutableList maskFieldPaths() { + return ImmutableList.of(protoPath); + } + + @Override + public ImmutableList maskDimensionNames() { + return ImmutableList.of(); + } + + @Override + public ImmutableList extract(DeviceInfo deviceInfo) { + return getter.apply(deviceInfo); + } + } + + private static final class DimensionSource extends DeviceInfoSource { + /** The one proto field that carries every device dimension, whichever dimension is named. */ + private static final String COMPOSITE_DIMENSION_PATH = "device_feature.composite_dimension"; + + private final String name; + + DimensionSource(String name) { + this.name = name; + } + + @Override + public ImmutableList maskFieldPaths() { + return ImmutableList.of(COMPOSITE_DIMENSION_PATH); + } + + @Override + public ImmutableList maskDimensionNames() { + return ImmutableList.of(name); + } + + @Override + public ImmutableList extract(DeviceInfo deviceInfo) { + DeviceCompositeDimension composite = deviceInfo.getDeviceFeature().getCompositeDimension(); + return Stream.concat( + composite.getSupportedDimensionList().stream(), + composite.getRequiredDimensionList().stream()) + .filter(dimension -> dimension.getName().equals(name)) + .map(DeviceDimension::getValue) + .collect(toImmutableList()); + } + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyDescriptor.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyDescriptor.java new file mode 100644 index 0000000000..e4c226dab9 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyDescriptor.java @@ -0,0 +1,96 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; + +/** + * The declarative metadata for one key usable in device search: either a device-native key or a + * host key projected (cross-entity) into device search. + * + *

The two source lists are the key's {@code GetLabInfo} footprint, kept as separate typed lists + * so mask derivation is fully type-safe (no dispatch): {@link #deviceInfoSources()} feed the {@code + * DeviceInfoMask} (device-native keys), {@link #labInfoSources()} feed the {@code LabInfoMask} (a + * projected host key reuses the host key's lab sources so the pull actually fetches the host field + * stamped onto each device). Both empty means the value comes entirely from a non-{@code + * GetLabInfo} feed (for example WiFi SSID from ConfigService) and contributes no mask. + * + *

A device key carries a single {@link #display()} (its device-search label). When a host + * attribute is shown in device search it is a projected key here with its own device-search name; + * the host descriptor keeps its host-search name. + * + *

{@link #isLongTail()} is {@code false} for a built-in key (declared in a catalog) and {@code + * true} for a key minted on demand by the registry for a discovered dimension or host property. It + * is stamped at creation and never hand-set at a call site: a built-in and a minted descriptor are + * the same type, and only the registry knows which is which, so it records that fact here. + */ +@AutoValue +public abstract class DeviceKeyDescriptor { + + /** The unique, type-safe identifier of the key (e.g. {@code "device_field::uuid"}). */ + public abstract String id(); + + /** {@code GetLabInfo} device-side sources; union drives the {@code DeviceInfoMask}. */ + public abstract ImmutableList deviceInfoSources(); + + /** + * {@code GetLabInfo} lab-side sources; union drives the {@code LabInfoMask}. Empty for a + * device-native key; populated for a projected host key (reusing the host key's lab sources). + */ + public abstract ImmutableList labInfoSources(); + + /** The device-search display (name + plural grammar). */ + public abstract KeyDisplay display(); + + /** Whether this descriptor was minted for a discovered (non-built-in) key. */ + public abstract boolean isLongTail(); + + /** Creates a builder for {@link DeviceKeyDescriptor}. */ + public static Builder builder() { + return new AutoValue_DeviceKeyDescriptor.Builder() + .setDeviceInfoSources(ImmutableList.of()) + .setLabInfoSources(ImmutableList.of()) + .setIsLongTail(false); + } + + /** Builder for {@link DeviceKeyDescriptor}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setId(String id); + + public abstract Builder setDeviceInfoSources(ImmutableList sources); + + /** Convenience for the common single device-source case. */ + public final Builder setDeviceInfoSource(DeviceInfoSource source) { + return setDeviceInfoSources(ImmutableList.of(source)); + } + + public abstract Builder setLabInfoSources(ImmutableList sources); + + /** Convenience for the common single lab-source case (a projected host key). */ + public final Builder setLabInfoSource(LabInfoSource source) { + return setLabInfoSources(ImmutableList.of(source)); + } + + public abstract Builder setDisplay(KeyDisplay display); + + public abstract Builder setIsLongTail(boolean isLongTail); + + public abstract DeviceKeyDescriptor build(); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyRegistry.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyRegistry.java new file mode 100644 index 0000000000..19a8090adc --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyRegistry.java @@ -0,0 +1,174 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQuery.Mask.DeviceInfoMask; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQuery.Mask.DeviceInfoMask.DimensionsMask; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQuery.Mask.LabInfoMask; +import com.google.protobuf.FieldMask; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; + +/** + * The per-deployment catalog of built-in keys usable in device search, and the factory/parser for + * every device-search key id (built-in or long-tail). + * + *

Its built-in seed is the universal common device-native keys plus the Group 1 host keys + * projected into device search; a subclass adds the extra keys its deployment reaches (WiFi SSID + * for standalone ATS; owners/executors and 1P host projections internally; the ATS controller in + * the partner aggregator). The full set of keys a user can actually reference is this built-in set + * plus the long-tail keys ({@code dimension::}, cross-entity {@code host_property::}) + * that are discovered from data and minted on demand by {@link #getKey}; long-tail keys are never + * in the built-in set and therefore never contribute to the derived core-pull mask. + * + *

Because the same {@link DeviceInfoSource} / {@link LabInfoSource} objects contribute to the + * mask and know how to extract their value, the {@code DeviceInfoMask} and {@code LabInfoMask} + * derived here cannot drift from extraction. + */ +public abstract class DeviceKeyRegistry { + + private static final ImmutableSet ALLOWED_PREFIXES = + ImmutableSet.of( + DeviceKeys.PREFIX_DEVICE_FIELD, + DeviceKeys.PREFIX_DIMENSION, + DeviceKeys.PREFIX_DEVICE_CONFIG, + HostKeys.PREFIX_HOST_FIELD, + HostKeys.PREFIX_HOST_PROPERTY); + + private final ImmutableMap builtInKeys; + + /** + * Composes the universal common device keys and projected common host keys with a subclass's + * extra keys. Fails fast on a duplicate id or an id with an unknown namespace prefix. + */ + protected DeviceKeyRegistry(ImmutableList extraKeys) { + ImmutableList all = + ImmutableList.builder() + .addAll(DeviceKeys.COMMON_DEVICE_KEYS) + .addAll(DeviceKeys.COMMON_HOST_PROJECTIONS) + .addAll(extraKeys) + .build(); + for (DeviceKeyDescriptor key : all) { + if (ALLOWED_PREFIXES.stream().noneMatch(key.id()::startsWith)) { + throw new IllegalArgumentException( + "Device key id '" + key.id() + "' must start with one of " + ALLOWED_PREFIXES); + } + } + // The two-arg collector throws IllegalArgumentException on a duplicate id (fail-fast). + this.builtInKeys = all.stream().collect(toImmutableMap(DeviceKeyDescriptor::id, key -> key)); + } + + /** + * Returns the descriptor for {@code keyId}: the built-in descriptor if registered, otherwise a + * minted long-tail descriptor for a {@code dimension::} or cross-entity {@code host_property::} + * id, otherwise empty. + */ + public Optional getKey(String keyId) { + DeviceKeyDescriptor builtIn = builtInKeys.get(keyId); + if (builtIn != null) { + return Optional.of(builtIn); + } + if (keyId.startsWith(DeviceKeys.PREFIX_DIMENSION)) { + return Optional.of( + DeviceKeys.longTailDimensionKey(keyId.substring(DeviceKeys.PREFIX_DIMENSION.length()))); + } + if (keyId.startsWith(HostKeys.PREFIX_HOST_PROPERTY)) { + return Optional.of(hostPropertyKey(keyId.substring(HostKeys.PREFIX_HOST_PROPERTY.length()))); + } + return Optional.empty(); + } + + /** Mints a long-tail device dimension key for a dimension discovered from data. */ + public DeviceKeyDescriptor dimensionKey(String dimensionName) { + return DeviceKeys.longTailDimensionKey(dimensionName); + } + + /** + * Mints a long-tail host-property key projected into device search (a cross-entity host attribute + * discovered from data). + */ + public DeviceKeyDescriptor hostPropertyKey(String propertyKey) { + return DeviceKeys.projectHostKey( + HostKeys.hostPropertyKey(propertyKey), KeyDisplay.of(propertyKey)); + } + + /** All built-in device key descriptors. */ + public ImmutableCollection builtInKeys() { + return builtInKeys.values(); + } + + /** All built-in device key ids. */ + public ImmutableSet builtInKeyIds() { + return builtInKeys.keySet(); + } + + /** Returns the display name for {@code keyId} (built-in or long-tail), or empty if unknown. */ + public Optional displayName(String keyId) { + return getKey(keyId).map(key -> key.display().name()); + } + + /** + * Derives the {@link DeviceInfoMask} from the union of every built-in device key's {@code + * deviceInfoSources}. Long-tail keys are excluded by construction (not in the built-in set). + */ + public DeviceInfoMask deriveDeviceInfoMask() { + Set fieldPaths = new LinkedHashSet<>(); + Set dimensionNames = new LinkedHashSet<>(); + for (DeviceKeyDescriptor key : builtInKeys.values()) { + for (DeviceInfoSource source : key.deviceInfoSources()) { + fieldPaths.addAll(source.maskFieldPaths()); + dimensionNames.addAll(source.maskDimensionNames()); + } + } + DeviceInfoMask.Builder mask = DeviceInfoMask.newBuilder(); + if (!fieldPaths.isEmpty()) { + mask.setFieldMask(FieldMask.newBuilder().addAllPaths(fieldPaths)); + } + if (!dimensionNames.isEmpty()) { + DimensionsMask dimsMask = + DimensionsMask.newBuilder().addAllDimensionNames(dimensionNames).build(); + mask.setSupportedDimensionsMask(dimsMask).setRequiredDimensionsMask(dimsMask); + } + return mask.build(); + } + + /** + * Derives the {@link LabInfoMask} from the union of every built-in device key's {@code + * labInfoSources} (the projected host keys), so the shared {@code GetLabInfo} pull fetches the + * host fields stamped onto each device. + */ + public LabInfoMask deriveLabInfoMask() { + Set fieldPaths = new LinkedHashSet<>(); + for (DeviceKeyDescriptor key : builtInKeys.values()) { + for (LabInfoSource source : key.labInfoSources()) { + fieldPaths.addAll(source.maskFieldPaths()); + } + } + LabInfoMask.Builder mask = LabInfoMask.newBuilder(); + if (!fieldPaths.isEmpty()) { + mask.setFieldMask(FieldMask.newBuilder().addAllPaths(fieldPaths)); + } + return mask.build(); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeys.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeys.java new file mode 100644 index 0000000000..d988381514 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeys.java @@ -0,0 +1,186 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import com.google.common.collect.ImmutableList; + +/** + * Standard open-source device key descriptors for MobileHarness device search. + * + *

Contains Group 1 (Universal Common Device Keys: typed {@code DeviceInfo} fields and core + * dimensions), the Group 1 host keys projected into device search, and Group 2 (Standalone ATS WiFi + * SSID). Each device-native key declares its {@code GetLabInfo} device sources, which give both the + * mask contribution and the raw extraction; WiFi SSID has no source because it is fed by + * ConfigService. + * + *

This catalog depends on {@link HostKeys}: a host attribute shown in device search is a + * projected key ({@link #projectHostKey}) that reuses the host key's id and lab sources, adding + * only a device-search display name. The dependency is one-way (host keys never reference device + * keys). + */ +public final class DeviceKeys { + + public static final String PREFIX_DEVICE_FIELD = "device_field::"; + public static final String PREFIX_DIMENSION = "dimension::"; + public static final String PREFIX_DEVICE_CONFIG = "device_config::"; + + // Group 1: Universal common device keys (typed DeviceInfo fields). + public static final DeviceKeyDescriptor UUID = + DeviceKeyDescriptor.builder() + .setId(PREFIX_DEVICE_FIELD + "uuid") + .setDeviceInfoSource( + DeviceInfoSource.field( + "device_locator.id", d -> nonEmpty(d.getDeviceLocator().getId()))) + .setDisplay(KeyDisplay.of("UUID")) + .build(); + + public static final DeviceKeyDescriptor STATUS = + DeviceKeyDescriptor.builder() + .setId(PREFIX_DEVICE_FIELD + "status") + .setDeviceInfoSource( + DeviceInfoSource.field( + "device_status", d -> ImmutableList.of(d.getDeviceStatus().name()))) + .setDisplay(KeyDisplay.of("Status")) + .build(); + + public static final DeviceKeyDescriptor TYPE = + DeviceKeyDescriptor.builder() + .setId(PREFIX_DEVICE_FIELD + "type") + .setDeviceInfoSource( + DeviceInfoSource.field( + "device_feature.type", + d -> ImmutableList.copyOf(d.getDeviceFeature().getTypeList()))) + .setDisplay(KeyDisplay.of("Type")) + .build(); + + public static final DeviceKeyDescriptor DRIVER = + DeviceKeyDescriptor.builder() + .setId(PREFIX_DEVICE_FIELD + "driver") + .setDeviceInfoSource( + DeviceInfoSource.field( + "device_feature.driver", + d -> ImmutableList.copyOf(d.getDeviceFeature().getDriverList()))) + .setDisplay(KeyDisplay.plural("Supported Drivers")) + .build(); + + public static final DeviceKeyDescriptor DECORATOR = + DeviceKeyDescriptor.builder() + .setId(PREFIX_DEVICE_FIELD + "decorator") + .setDeviceInfoSource( + DeviceInfoSource.field( + "device_feature.decorator", + d -> ImmutableList.copyOf(d.getDeviceFeature().getDecoratorList()))) + .setDisplay(KeyDisplay.plural("Supported Decorators")) + .build(); + + // Group 1: Universal common core dimensions. + public static final DeviceKeyDescriptor MODEL = dimensionKey("model", KeyDisplay.of("Model")); + public static final DeviceKeyDescriptor OS = dimensionKey("os", KeyDisplay.of("OS")); + public static final DeviceKeyDescriptor SDK_VERSION = + dimensionKey("sdk_version", KeyDisplay.of("SDK Version")); + public static final DeviceKeyDescriptor SOFTWARE_VERSION = + dimensionKey("software_version", KeyDisplay.of("Software Version")); + public static final DeviceKeyDescriptor DEVICE_FORM = + dimensionKey("device_form", KeyDisplay.of("Form")); + public static final DeviceKeyDescriptor DEVICE_CLASS_NAME = + dimensionKey("device_class_name", KeyDisplay.of("Device Class")); + public static final DeviceKeyDescriptor MANUFACTURER = + dimensionKey("manufacturer", KeyDisplay.of("Manufacturer")); + + /** Standard Group 1 common device-native keys (present in every deployment). */ + public static final ImmutableList COMMON_DEVICE_KEYS = + ImmutableList.of( + UUID, + STATUS, + TYPE, + DRIVER, + DECORATOR, + MODEL, + OS, + SDK_VERSION, + SOFTWARE_VERSION, + DEVICE_FORM, + DEVICE_CLASS_NAME, + MANUFACTURER); + + /** + * Group 1 host keys projected into device search (cross-entity host attributes stamped onto each + * device). {@code device_count} is deliberately not projected: it is a host-only numeric key. + */ + public static final ImmutableList COMMON_HOST_PROJECTIONS = + ImmutableList.of( + projectHostKey(HostKeys.HOST_NAME, KeyDisplay.of("Host Name")), + projectHostKey(HostKeys.HOST_IP, KeyDisplay.of("Host IP")), + projectHostKey(HostKeys.CONNECTIVITY, KeyDisplay.of("Host Lab Server Connectivity")), + projectHostKey(HostKeys.HOST_OS, KeyDisplay.of("Host OS")), + projectHostKey(HostKeys.LAB_SERVER_VERSION, KeyDisplay.of("Host Lab Server Version"))); + + // The helpers below are package-private by design. They are catalog-authoring helpers shared + // across the schema catalogs (DeviceKeys, AtsDeviceKeys, InternalDeviceKeys, + // PartnerAtsDeviceKeys) + // and the registry. They are deliberately NOT public: the sole public way to obtain a key is + // through the registry (DeviceKeyRegistry#getKey / #dimensionKey / #hostPropertyKey), so no + // caller + // can hand-build or mint a key while bypassing the registry, which is the single key-id + // authority. + + /** Builds a built-in device key backed by a single named composite dimension. */ + static DeviceKeyDescriptor dimensionKey(String dimensionName, KeyDisplay display) { + return DeviceKeyDescriptor.builder() + .setId(PREFIX_DIMENSION + dimensionName) + .setDeviceInfoSource(DeviceInfoSource.dimension(dimensionName)) + .setDisplay(display) + .build(); + } + + /** + * Builds a long-tail device key for a dimension discovered from data. Package-private on purpose: + * this is the implementation the registry delegates to. Callers mint long-tail keys through the + * public {@link DeviceKeyRegistry#dimensionKey(String)} (or {@link + * DeviceKeyRegistry#getKey(String)}), never this static, so the registry stays the single choke + * point that stamps {@code isLongTail} and enforces scope. The minted key carries no curated + * display (raw name) and is flagged long-tail. + */ + static DeviceKeyDescriptor longTailDimensionKey(String dimensionName) { + return DeviceKeyDescriptor.builder() + .setId(PREFIX_DIMENSION + dimensionName) + .setDeviceInfoSource(DeviceInfoSource.dimension(dimensionName)) + .setDisplay(KeyDisplay.of(dimensionName)) + .setIsLongTail(true) + .build(); + } + + /** + * Projects a host key into device search: same id and lab sources, a device-search display name. + * Preserves the host key's long-tail flag so a projected discovered host property stays + * long-tail. + */ + static DeviceKeyDescriptor projectHostKey(HostKeyDescriptor host, KeyDisplay deviceDisplay) { + return DeviceKeyDescriptor.builder() + .setId(host.id()) + .setLabInfoSources(host.labInfoSources()) + .setDisplay(deviceDisplay) + .setIsLongTail(host.isLongTail()) + .build(); + } + + private static ImmutableList nonEmpty(String value) { + return value.isEmpty() ? ImmutableList.of() : ImmutableList.of(value); + } + + private DeviceKeys() {} +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyDescriptor.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyDescriptor.java new file mode 100644 index 0000000000..94f171013f --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyDescriptor.java @@ -0,0 +1,77 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; + +/** + * The declarative metadata for one key usable in host search. + * + *

A host key has a single source list, {@link #labInfoSources()} (a host key never reads + * device-side proto), whose union drives the {@code LabInfoMask}. Empty means the value comes + * entirely from a non-{@code GetLabInfo} feed (for example release status from HostInfoService, or + * {@code ats_controller} from fan-out provenance) and contributes no mask. + * + *

A host key carries a single {@link #display()} (its host-search label). When a host attribute + * is also wanted in device search it is projected as a separate {@link DeviceKeyDescriptor} with + * its own device-search name; the host descriptor never carries a device-search name. + * + *

{@link #isLongTail()} is {@code false} for a built-in host key and {@code true} for a host + * property minted on demand by the registry. See {@link DeviceKeyDescriptor#isLongTail()}. + */ +@AutoValue +public abstract class HostKeyDescriptor { + + /** The unique, type-safe identifier of the host key (e.g. {@code "host_field::host_name"}). */ + public abstract String id(); + + /** {@code GetLabInfo} lab-side sources; union drives the {@code LabInfoMask}. */ + public abstract ImmutableList labInfoSources(); + + /** The host-search display (name + plural grammar). */ + public abstract KeyDisplay display(); + + /** Whether this descriptor was minted for a discovered (non-built-in) host property. */ + public abstract boolean isLongTail(); + + /** Creates a builder for {@link HostKeyDescriptor}. */ + public static Builder builder() { + return new AutoValue_HostKeyDescriptor.Builder() + .setLabInfoSources(ImmutableList.of()) + .setIsLongTail(false); + } + + /** Builder for {@link HostKeyDescriptor}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setId(String id); + + public abstract Builder setLabInfoSources(ImmutableList sources); + + /** Convenience for the common single lab-source case. */ + public final Builder setLabInfoSource(LabInfoSource source) { + return setLabInfoSources(ImmutableList.of(source)); + } + + public abstract Builder setDisplay(KeyDisplay display); + + public abstract Builder setIsLongTail(boolean isLongTail); + + public abstract HostKeyDescriptor build(); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyRegistry.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyRegistry.java new file mode 100644 index 0000000000..1a8ea13695 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyRegistry.java @@ -0,0 +1,119 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQuery.Mask.LabInfoMask; +import com.google.protobuf.FieldMask; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; + +/** + * The per-deployment catalog of built-in keys usable in host search, and the factory/parser for + * every host-search key id (built-in or long-tail). + * + *

Its built-in seed is the universal common host keys; a subclass adds the extra keys its + * deployment reaches (release/daemon/lab-type internally; the ATS controller in the partner + * aggregator). The full set of keys a user can reference is this built-in set plus the long-tail + * {@code host_property::} keys discovered from data and minted on demand by {@link #getKey}; + * long-tail keys are never in the built-in set and never contribute to the derived mask. + */ +public abstract class HostKeyRegistry { + + private static final ImmutableSet ALLOWED_PREFIXES = + ImmutableSet.of(HostKeys.PREFIX_HOST_FIELD, HostKeys.PREFIX_HOST_PROPERTY); + + private final ImmutableMap builtInKeys; + + /** + * Composes the universal common host keys with a subclass's extra keys. Fails fast on a duplicate + * id or an id with an unknown namespace prefix. + */ + protected HostKeyRegistry(ImmutableList extraKeys) { + ImmutableList all = + ImmutableList.builder() + .addAll(HostKeys.COMMON_HOST_KEYS) + .addAll(extraKeys) + .build(); + for (HostKeyDescriptor key : all) { + if (ALLOWED_PREFIXES.stream().noneMatch(key.id()::startsWith)) { + throw new IllegalArgumentException( + "Host key id '" + key.id() + "' must start with one of " + ALLOWED_PREFIXES); + } + } + this.builtInKeys = all.stream().collect(toImmutableMap(HostKeyDescriptor::id, key -> key)); + } + + /** + * Returns the descriptor for {@code keyId}: the built-in descriptor if registered, otherwise a + * minted long-tail descriptor for a {@code host_property::} id, otherwise empty. + */ + public Optional getKey(String keyId) { + HostKeyDescriptor builtIn = builtInKeys.get(keyId); + if (builtIn != null) { + return Optional.of(builtIn); + } + if (keyId.startsWith(HostKeys.PREFIX_HOST_PROPERTY)) { + return Optional.of(hostPropertyKey(keyId.substring(HostKeys.PREFIX_HOST_PROPERTY.length()))); + } + return Optional.empty(); + } + + /** Mints a long-tail host-property key for a property discovered from data. */ + public HostKeyDescriptor hostPropertyKey(String propertyKey) { + return HostKeys.hostPropertyKey(propertyKey); + } + + /** All built-in host key descriptors. */ + public ImmutableCollection builtInKeys() { + return builtInKeys.values(); + } + + /** All built-in host key ids. */ + public ImmutableSet builtInKeyIds() { + return builtInKeys.keySet(); + } + + /** Returns the display name for {@code keyId} (built-in or long-tail), or empty if unknown. */ + public Optional displayName(String keyId) { + return getKey(keyId).map(key -> key.display().name()); + } + + /** + * Derives the {@link LabInfoMask} from the union of every built-in host key's {@code + * labInfoSources}. Long-tail keys are excluded by construction (not in the built-in set). + */ + public LabInfoMask deriveLabInfoMask() { + Set fieldPaths = new LinkedHashSet<>(); + for (HostKeyDescriptor key : builtInKeys.values()) { + for (LabInfoSource source : key.labInfoSources()) { + fieldPaths.addAll(source.maskFieldPaths()); + } + } + LabInfoMask.Builder mask = LabInfoMask.newBuilder(); + if (!fieldPaths.isEmpty()) { + mask.setFieldMask(FieldMask.newBuilder().addAllPaths(fieldPaths)); + } + return mask.build(); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeys.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeys.java new file mode 100644 index 0000000000..acc409bc17 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeys.java @@ -0,0 +1,106 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import com.google.common.collect.ImmutableList; + +/** + * Standard open-source host key descriptors for MobileHarness host search. + * + *

Contains Group 1 (Universal Common Host Keys). Each key declares its {@code GetLabInfo} lab + * sources, which give both the mask contribution and the raw extraction. {@code device_count} has + * no source because it is synthesized from the device list. Display transforms, such as bucketing a + * lab status into a connectivity label, are a key-level combiner concern handled at index time. + * + *

Display names are the host-search labels (spec §0 host table). When a host attribute is also + * shown in device search, {@link DeviceKeys} projects it with a device-search name; the projection + * reuses the id and lab sources declared here. + */ +public final class HostKeys { + + public static final String PREFIX_HOST_FIELD = "host_field::"; + public static final String PREFIX_HOST_PROPERTY = "host_property::"; + + public static final HostKeyDescriptor HOST_NAME = + HostKeyDescriptor.builder() + .setId(PREFIX_HOST_FIELD + "host_name") + .setLabInfoSource( + LabInfoSource.field( + "lab_locator.host_name", li -> nonEmpty(li.getLabLocator().getHostName()))) + .setDisplay(KeyDisplay.of("Host Name")) + .build(); + + public static final HostKeyDescriptor HOST_IP = + HostKeyDescriptor.builder() + .setId(PREFIX_HOST_FIELD + "host_ip") + .setLabInfoSource( + LabInfoSource.field("lab_locator.ip", li -> nonEmpty(li.getLabLocator().getIp()))) + .setDisplay(KeyDisplay.of("Host IP")) + .build(); + + public static final HostKeyDescriptor CONNECTIVITY = + HostKeyDescriptor.builder() + .setId(PREFIX_HOST_FIELD + "connectivity") + .setLabInfoSource( + LabInfoSource.field("lab_status", li -> ImmutableList.of(li.getLabStatus().name()))) + .setDisplay(KeyDisplay.of("Lab Server Connectivity")) + .build(); + + public static final HostKeyDescriptor HOST_OS = + HostKeyDescriptor.builder() + .setId(PREFIX_HOST_PROPERTY + "host_os") + .setLabInfoSource(LabInfoSource.hostProperty("host_os")) + .setDisplay(KeyDisplay.of("Host OS")) + .build(); + + public static final HostKeyDescriptor LAB_SERVER_VERSION = + HostKeyDescriptor.builder() + .setId(PREFIX_HOST_FIELD + "lab_server_version") + .setLabInfoSource(LabInfoSource.hostProperty("host_version")) + .setDisplay(KeyDisplay.of("Lab Server Version")) + .build(); + + /** Synthesized from the device list, so it contributes no mask. Host search only. */ + public static final HostKeyDescriptor DEVICE_COUNT = + HostKeyDescriptor.builder() + .setId(PREFIX_HOST_FIELD + "device_count") + .setDisplay(KeyDisplay.of("Device Count")) + .build(); + + /** Standard Group 1 common host keys (present in every deployment). */ + public static final ImmutableList COMMON_HOST_KEYS = + ImmutableList.of(HOST_NAME, HOST_IP, CONNECTIVITY, HOST_OS, LAB_SERVER_VERSION, DEVICE_COUNT); + + /** + * Builds a long-tail host-property key for {@code key} discovered from data. The registry mints + * these on demand; they carry no curated display (raw name) and are flagged long-tail. + */ + static HostKeyDescriptor hostPropertyKey(String key) { + return HostKeyDescriptor.builder() + .setId(PREFIX_HOST_PROPERTY + key) + .setLabInfoSource(LabInfoSource.hostProperty(key)) + .setDisplay(KeyDisplay.of(key)) + .setIsLongTail(true) + .build(); + } + + private static ImmutableList nonEmpty(String value) { + return value.isEmpty() ? ImmutableList.of() : ImmutableList.of(value); + } + + private HostKeys() {} +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/KeyDisplay.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/KeyDisplay.java new file mode 100644 index 0000000000..811c0d87b0 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/KeyDisplay.java @@ -0,0 +1,54 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import com.google.auto.value.AutoValue; + +/** + * The user-facing label for a search key in one search context, plus its grammatical number. + * + *

The {@link #name()} is the display shown in every surface (search-bar suggestions, column + * selector, column header, value list). For a built-in key it is the curated name (for example + * {@code "Model"}); for a long-tail key it is the raw dimension or host-property name. The one + * surface that differs is the value-picker (chip-detail) title, which prepends a {@code "Dimension + * "} / {@code "Host Property "} category prefix for long-tail keys only; that prefix is derived + * from the key id namespace at render time, so it is not stored here. + * + *

{@link #isPlural()} drives the value-picker polarity grammar ({@code "are"} vs {@code "is"}). + * It is a hand-picked display attribute, true only for keys whose label reads as a plural noun + * (Owners, Supported Drivers, Supported Decorators, Executors); it is not the same axis as whether + * a device can carry multiple values. + */ +@AutoValue +public abstract class KeyDisplay { + + /** The display name (curated for a built-in key, the raw key name for a long-tail key). */ + public abstract String name(); + + /** Whether the label is grammatically plural, e.g. "Owners are" vs "Model is". */ + public abstract boolean isPlural(); + + /** A singular-grammar display name (for example {@code "Model"}). */ + public static KeyDisplay of(String name) { + return new AutoValue_KeyDisplay(name, /* isPlural= */ false); + } + + /** A plural-grammar display name (for example {@code "Owners"}). */ + public static KeyDisplay plural(String name) { + return new AutoValue_KeyDisplay(name, /* isPlural= */ true); + } +} diff --git a/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/LabInfoSource.java b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/LabInfoSource.java new file mode 100644 index 0000000000..f54ad73ec5 --- /dev/null +++ b/src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema/LabInfoSource.java @@ -0,0 +1,105 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.mobileharness.api.model.proto.Lab.HostProperty; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabInfo; +import java.util.function.Function; + +/** + * A single component of a host key's value that lives inside the {@code GetLabInfo} response and + * therefore participates in the {@code LabInfoMask}. + * + *

See {@link DeviceInfoSource} for the rationale: a source declares its mask contribution and + * its raw field extraction so the two cannot drift. Non-{@code GetLabInfo} host data (HostInfo, + * provenance) is not a source. + */ +public abstract class LabInfoSource { + + /** The {@code LabInfoMask.field_mask} paths this source requires. */ + public abstract ImmutableList maskFieldPaths(); + + /** Reads the raw value(s) of this source from a {@code LabInfo}. */ + public abstract ImmutableList extract(LabInfo labInfo); + + /** + * A typed field on {@code LabInfo}, named by its {@code LabInfoMask} path (for example {@code + * "lab_locator.host_name"}). The path drives the mask and the getter reads the value; both are + * given together at the call site, so a reader sees at a glance that they agree. Every derived + * mask is checked against the proto descriptor in the registry tests, so a mistyped path fails + * there rather than silently yielding empty values at serving time. + */ + public static LabInfoSource field( + String protoPath, Function> getter) { + return new FieldSource(protoPath, getter); + } + + /** + * A property key in {@code lab_server_feature.host_properties}. All properties are pulled by one + * blanket mask path, so the mask contribution is the same regardless of the specific key. + */ + public static LabInfoSource hostProperty(String key) { + return new HostPropertySource(key); + } + + private static final class FieldSource extends LabInfoSource { + private final String protoPath; + private final Function> getter; + + FieldSource(String protoPath, Function> getter) { + this.protoPath = protoPath; + this.getter = getter; + } + + @Override + public ImmutableList maskFieldPaths() { + return ImmutableList.of(protoPath); + } + + @Override + public ImmutableList extract(LabInfo labInfo) { + return getter.apply(labInfo); + } + } + + private static final class HostPropertySource extends LabInfoSource { + /** The one proto field that carries every host property, whichever property key is named. */ + private static final String HOST_PROPERTIES_PATH = "lab_server_feature.host_properties"; + + private final String key; + + HostPropertySource(String key) { + this.key = key; + } + + @Override + public ImmutableList maskFieldPaths() { + return ImmutableList.of(HOST_PROPERTIES_PATH); + } + + @Override + public ImmutableList extract(LabInfo labInfo) { + return labInfo.getLabServerFeature().getHostProperties().getHostPropertyList().stream() + .filter(property -> property.getKey().equals(key)) + .map(HostProperty::getValue) + .collect(toImmutableList()); + } + } +} diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/BUILD b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/BUILD index b477eba96a..40843d9b09 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/BUILD +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/BUILD @@ -32,6 +32,7 @@ java_library( "//src/java/com/google/devtools/mobileharness/fe/v6/service/search:search_service_logic_impl", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index_builder", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:fleet_data_source", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/query:scenario_curation_module", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:fleet_snapshot_store", "//src/javatests/com/google/devtools/mobileharness/builddefs:truth", diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImplTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImplTest.java index 025fb28095..3913fd338e 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImplTest.java +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/SearchServiceLogicImplTest.java @@ -44,11 +44,13 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.SearchEntity; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndexBuilder; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.FleetDataSource; import com.google.devtools.mobileharness.fe.v6.service.search.query.ScenarioCurationModule; import com.google.devtools.mobileharness.fe.v6.service.search.refresh.FleetSnapshotStore; import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; +import com.google.inject.multibindings.MapBinder; import java.time.Instant; import org.junit.Before; import org.junit.Test; @@ -84,6 +86,7 @@ public void setUp() { @Override protected void configure() { bind(ListeningExecutorService.class).toInstance(newDirectExecutorService()); + MapBinder.newMapBinder(binder(), Fleet.class, FleetDataSource.class); } }); FleetSnapshotStore store = injector.getInstance(FleetSnapshotStore.class); diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD index a90884599b..26dc23d8da 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/BUILD @@ -28,13 +28,21 @@ java_library( "//src/devtools/mobileharness/api/model/proto:device_java_proto", "//src/devtools/mobileharness/api/model/proto:lab_java_proto", "//src/devtools/mobileharness/api/query/proto:lab_query_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:composite_fleet_index", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:composite_postings", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:device_enrichment", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:dimension_overlay", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index_builder", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_raw_data", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:host_enrichment", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:key_count", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:lazy_postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:overlay_view", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:value_key_pair", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:dimension_overlay_raw", "//src/javatests/com/google/devtools/mobileharness/builddefs:truth", "@maven//:com_google_guava_guava", "@maven//:junit_junit", diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositeFleetIndexTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositeFleetIndexTest.java new file mode 100644 index 0000000000..c91fdf8083 --- /dev/null +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositeFleetIndexTest.java @@ -0,0 +1,178 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.HashMap; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CompositeFleetIndex}. */ +@RunWith(JUnit4.class) +public final class CompositeFleetIndexTest { + + private CoreFleetIndex core; + private FakeOverlayView overlay; + private CompositeFleetIndex composite; + + @Before + public void setUp() { + core = + CoreFleetIndex.builder() + .setKeyIds(ImmutableSet.of("field::status", "dim::model")) + .setSortedValuesMap( + ImmutableMap.of( + "field::status", ImmutableList.of("busy", "idle"), + "dim::model", ImmutableList.of("pixel 7", "pixel 8"))) + .setValueCountsMap( + ImmutableMap.of( + "field::status", ImmutableMap.of("idle", 10, "busy", 5), + "dim::model", ImmutableMap.of("pixel 8", 8, "pixel 7", 7))) + .setValueDisplaysMap( + ImmutableMap.of( + "field::status", ImmutableMap.of("idle", "IDLE", "busy", "BUSY"), + "dim::model", ImmutableMap.of("pixel 8", "Pixel 8", "pixel 7", "Pixel 7"))) + .setDisplayNamesMap(ImmutableMap.of("field::status", "Status", "dim::model", "Model")) + .setSemanticGlobalSorted( + ImmutableList.of( + new ValueKeyPair("pixel 7", "dim::model"), + new ValueKeyPair("pixel 8", "dim::model"))) + .setGlobalExact( + ImmutableMap.of("pixel 8", ImmutableList.of(new KeyCount("dim::model", 8)))) + .build(); + + overlay = new FakeOverlayView(); + overlay.loadedKeys = ImmutableSet.of("dim::carrier", "dim::battery_level"); + overlay.sortedValuesMap.put("dim::carrier", ImmutableList.of("att", "t-mobile", "verizon")); + overlay.valueCountsMap.put("dim::carrier", ImmutableMap.of("verizon", 1200, "att", 500)); + overlay.valueDisplaysMap.put( + "dim::carrier", ImmutableMap.of("verizon", "Verizon", "att", "AT&T")); + + composite = new CompositeFleetIndex(core, overlay); + } + + @Test + public void coreKey_delegatesToCore() { + assertThat(composite.sortedValues("field::status")).containsExactly("busy", "idle").inOrder(); + assertThat(composite.valueCounts("field::status")).containsEntry("idle", 10); + assertThat(composite.valueDisplays("field::status")).containsEntry("idle", "IDLE"); + assertThat(composite.displayName("field::status")).isEqualTo("Status"); + assertThat(composite.valueCount("field::status", "idle")).isEqualTo(10); + } + + @Test + public void overlayKey_delegatesToOverlay() { + assertThat(composite.sortedValues("dim::carrier")) + .containsExactly("att", "t-mobile", "verizon") + .inOrder(); + assertThat(composite.valueCounts("dim::carrier")).containsEntry("verizon", 1200); + assertThat(composite.valueDisplays("dim::carrier")).containsEntry("verizon", "Verizon"); + assertThat(composite.displayName("dim::carrier")).isEqualTo("Dimension carrier"); + assertThat(composite.valueCount("dim::carrier", "verizon")).isEqualTo(1200); + } + + @Test + public void absentKey_returnsEmptyAndDerivesDisplayName() { + assertThat(composite.sortedValues("dim::unknown")).isEmpty(); + assertThat(composite.valueCounts("dim::unknown")).isEmpty(); + assertThat(composite.valueDisplays("dim::unknown")).isEmpty(); + assertThat(composite.displayName("dim::unknown")).isEqualTo("Dimension unknown"); + assertThat(composite.displayName("prop::rack")).isEqualTo("Host Property rack"); + assertThat(composite.valueCount("dim::unknown", "val")).isEqualTo(0); + } + + @Test + public void keyIds_returnsUnion() { + assertThat(composite.keyIds()) + .containsExactly("field::status", "dim::model", "dim::carrier", "dim::battery_level"); + } + + @Test + public void globalIndices_delegateToCoreOnly() { + // Invariant D6: global bare-value search is strictly isolated from overlay data. + assertThat(composite.semanticGlobalSorted()) + .containsExactly( + new ValueKeyPair("pixel 7", "dim::model"), new ValueKeyPair("pixel 8", "dim::model")) + .inOrder(); + assertThat(composite.globalExact()) + .containsExactly("pixel 8", ImmutableList.of(new KeyCount("dim::model", 8))); + } + + private static final class FakeOverlayView implements OverlayView { + ImmutableSet loadedKeys = ImmutableSet.of(); + final Map> sortedValuesMap = new HashMap<>(); + final Map> valueCountsMap = new HashMap<>(); + final Map> valueDisplaysMap = new HashMap<>(); + + @Override + public boolean containsKey(String keyId) { + return loadedKeys.contains(keyId); + } + + @Override + public ImmutableSet loadedKeys() { + return loadedKeys; + } + + @Override + public ImmutableList sortedValues(String keyId) { + return sortedValuesMap.getOrDefault(keyId, ImmutableList.of()); + } + + @Override + public ImmutableMap valueCounts(String keyId) { + return valueCountsMap.getOrDefault(keyId, ImmutableMap.of()); + } + + @Override + public ImmutableMap valueDisplays(String keyId) { + return valueDisplaysMap.getOrDefault(keyId, ImmutableMap.of()); + } + + @Override + public int valueCount(String keyId, String value) { + return valueCounts(keyId).getOrDefault(value, 0); + } + + @Override + public int[] getPostings(String keyId, String value) { + return new int[0]; + } + + @Override + public ImmutableMap postingsForKey(String keyId) { + return ImmutableMap.of(); + } + + @Override + public ImmutableSet valuesForKey(int deviceIndex, String keyId) { + return ImmutableSet.of(); + } + + @Override + public ImmutableList displayValues(int deviceIndex, String keyId) { + return ImmutableList.of(); + } + } +} diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositePostingsTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositePostingsTest.java new file mode 100644 index 0000000000..58cdc8ba4f --- /dev/null +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/CompositePostingsTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.HashMap; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CompositePostings}. */ +@RunWith(JUnit4.class) +public final class CompositePostingsTest { + + private FakePostings corePostings; + private FakeOverlayView overlayView; + private CompositePostings composite; + + @Before + public void setUp() { + corePostings = new FakePostings(); + corePostings.postingsMap.put( + "field::status", ImmutableMap.of("idle", new int[] {0, 2}, "busy", new int[] {1})); + + overlayView = new FakeOverlayView(); + overlayView.loadedKeys = ImmutableSet.of("dim::carrier"); + overlayView.postingsMap.put( + "dim::carrier", ImmutableMap.of("verizon", new int[] {0, 1}, "att", new int[] {2})); + + composite = new CompositePostings(corePostings, overlayView); + } + + @Test + public void coreKey_delegatesToCore() { + assertThat(composite.get("field::status", "idle")).asList().containsExactly(0, 2).inOrder(); + assertThat(composite.forKey("field::status")).containsKey("idle"); + } + + @Test + public void overlayKey_delegatesToOverlay() { + assertThat(composite.get("dim::carrier", "verizon")).asList().containsExactly(0, 1).inOrder(); + assertThat(composite.forKey("dim::carrier")).containsKey("verizon"); + } + + @Test + public void absentKey_returnsEmpty() { + assertThat(composite.get("dim::unknown", "val")).isEmpty(); + assertThat(composite.forKey("dim::unknown")).isEmpty(); + } + + private static final class FakePostings implements Postings { + final Map> postingsMap = new HashMap<>(); + + @Override + public int[] get(String keyId, String value) { + ImmutableMap keyPostings = postingsMap.get(keyId); + if (keyPostings != null) { + int[] posting = keyPostings.get(value); + if (posting != null) { + return posting; + } + } + return new int[0]; + } + + @Override + public ImmutableMap forKey(String keyId) { + ImmutableMap keyPostings = postingsMap.get(keyId); + return keyPostings != null ? keyPostings : ImmutableMap.of(); + } + } + + private static final class FakeOverlayView implements OverlayView { + ImmutableSet loadedKeys = ImmutableSet.of(); + final Map> postingsMap = new HashMap<>(); + + @Override + public boolean containsKey(String keyId) { + return loadedKeys.contains(keyId); + } + + @Override + public ImmutableSet loadedKeys() { + return loadedKeys; + } + + @Override + public ImmutableList sortedValues(String keyId) { + return ImmutableList.of(); + } + + @Override + public ImmutableMap valueCounts(String keyId) { + return ImmutableMap.of(); + } + + @Override + public ImmutableMap valueDisplays(String keyId) { + return ImmutableMap.of(); + } + + @Override + public int valueCount(String keyId, String value) { + return 0; + } + + @Override + public int[] getPostings(String keyId, String value) { + ImmutableMap keyPostings = postingsMap.get(keyId); + if (keyPostings != null) { + int[] posting = keyPostings.get(value); + if (posting != null) { + return posting; + } + } + return new int[0]; + } + + @Override + public ImmutableMap postingsForKey(String keyId) { + ImmutableMap keyPostings = postingsMap.get(keyId); + return keyPostings != null ? keyPostings : ImmutableMap.of(); + } + + @Override + public ImmutableSet valuesForKey(int deviceIndex, String keyId) { + return ImmutableSet.of(); + } + + @Override + public ImmutableList displayValues(int deviceIndex, String keyId) { + return ImmutableList.of(); + } + } +} diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlayTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlayTest.java new file mode 100644 index 0000000000..2bae2c51c8 --- /dev/null +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/DimensionOverlayTest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.index; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.DimensionOverlayRaw; +import java.time.Instant; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link DimensionOverlay}. */ +@RunWith(JUnit4.class) +public final class DimensionOverlayTest { + + @Test + public void create_and_bind_reindexesAccurately() { + DimensionOverlayRaw raw = + DimensionOverlayRaw.create( + "dim::carrier", + ImmutableMap.of( + "uuid-1", ImmutableList.of("Verizon"), + "uuid-2", ImmutableList.of("T-Mobile", "AT&T"), + "uuid-3", ImmutableList.of("Verizon"))); + + FleetSnapshot snapshotA = + FleetSnapshot.builder() + .setBuildTime(Instant.ofEpochSecond(1000)) + .setDevices(ImmutableList.of()) + .setHosts(ImmutableList.of()) + .setIndex(CoreFleetIndex.empty()) + .setHostIndex(CoreFleetIndex.empty()) + .setUuidToIndex(ImmutableMap.of("uuid-1", 0, "uuid-2", 1, "uuid-3", 2)) + .build(); + + DimensionOverlay overlay = DimensionOverlay.create(raw, snapshotA); + + assertThat(overlay.keyId()).isEqualTo("dim::carrier"); + assertThat(overlay.sortedValues()).containsExactly("at&t", "t-mobile", "verizon").inOrder(); + assertThat(overlay.valueCounts()).containsEntry("verizon", 2); + assertThat(overlay.valueDisplays()).containsEntry("verizon", "Verizon"); + + // Check postings bound to Snapshot A + DimensionOverlay.SnapshotBoundPostings boundA = overlay.bind(snapshotA); + assertThat(boundA.get("verizon")).asList().containsExactly(0, 2).inOrder(); + assertThat(boundA.get("t-mobile")).asList().containsExactly(1); + assertThat(boundA.get("at&t")).asList().containsExactly(1); + + // Snapshot B: uuid-1 moved to index 5, uuid-3 moved to index 1, uuid-2 is decommissioned + // (absent) + FleetSnapshot snapshotB = + FleetSnapshot.builder() + .setBuildTime(Instant.ofEpochSecond(2000)) + .setDevices(ImmutableList.of()) + .setHosts(ImmutableList.of()) + .setIndex(CoreFleetIndex.empty()) + .setHostIndex(CoreFleetIndex.empty()) + .setUuidToIndex(ImmutableMap.of("uuid-1", 5, "uuid-3", 1)) + .build(); + + DimensionOverlay.SnapshotBoundPostings boundB = overlay.bind(snapshotB); + assertThat(boundB.get("verizon")).asList().containsExactly(1, 5).inOrder(); + assertThat(boundB.get("t-mobile")).isEmpty(); // uuid-2 was dropped safely + } +} diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilderTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilderTest.java index 0eeac3d6a5..18f7bc5cd1 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilderTest.java +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/index/FleetIndexBuilderTest.java @@ -134,11 +134,11 @@ public void build_indexesDevicesHostsAndValues() { assertThat(posting(postings, "dim::model", "pixel")).containsExactly(0); // Sorted distinct values. - assertThat(index.sortedValues().get("dim::os")).containsExactly("android", "ios").inOrder(); - assertThat(index.sortedValues().get("field::status")).containsExactly("busy", "idle").inOrder(); + assertThat(index.sortedValues("dim::os")).containsExactly("android", "ios").inOrder(); + assertThat(index.sortedValues("field::status")).containsExactly("busy", "idle").inOrder(); // Original casing preserved for display. - assertThat(index.valueDisplays().get("field::status")).containsEntry("idle", "IDLE"); + assertThat(index.valueDisplays("field::status")).containsEntry("idle", "IDLE"); // Key catalog: dimension and property keys discovered from data; display names resolved. assertThat(index.keyIds()) @@ -154,11 +154,11 @@ public void build_indexesDevicesHostsAndValues() { "prop::host_version", "host::host_name", "host::host_ip"); - assertThat(index.displayNames()).containsEntry("field::status", "Status"); - assertThat(index.displayNames()).containsEntry("dim::os", "OS"); - assertThat(index.displayNames()).containsEntry("dim::model", "Model"); + assertThat(index.displayName("field::status")).isEqualTo("Status"); + assertThat(index.displayName("dim::os")).isEqualTo("OS"); + assertThat(index.displayName("dim::model")).isEqualTo("Model"); // Discovered non-built-in keys derive their display name from the raw name. - assertThat(index.displayNames()).containsEntry("prop::location", "Host Property location"); + assertThat(index.displayName("prop::location")).isEqualTo("Host Property location"); // Device enrichment: wifi_ssid is single-valued, indexed under its normalized term with the // original casing preserved for display. @@ -166,17 +166,17 @@ public void build_indexesDevicesHostsAndValues() { assertThat(index.valueCount("config::wifi_ssid", "googleguest")).isEqualTo(1); LazyPostings wifiPostings = new LazyPostings(snapshot.devices()); assertThat(posting(wifiPostings, "config::wifi_ssid", "googleguest")).containsExactly(0); - assertThat(index.valueDisplays().get("config::wifi_ssid")) + assertThat(index.valueDisplays("config::wifi_ssid")) .containsEntry("googleguest", "GoogleGuest"); - assertThat(index.displayNames()).containsEntry("config::wifi_ssid", "Wi-Fi SSID"); + assertThat(index.displayName("config::wifi_ssid")).isEqualTo("Wi-Fi SSID"); // Host enrichment: the lab type is computed from the release enum (SHARED_LAB maps to Core) and // stamped, as a display name, on every device of the enriched host (device-0 and device-1 on // lab1), and absent for devices on an unenriched host (device-2). assertThat(index.valueCount("host::lab_type", "core lab")).isEqualTo(2); - assertThat(index.sortedValues().get("host::lab_type")).containsExactly("core lab"); - assertThat(index.valueDisplays().get("host::lab_type")).containsEntry("core lab", "Core Lab"); - assertThat(index.displayNames()).containsEntry("host::lab_type", "Host Lab Type"); + assertThat(index.sortedValues("host::lab_type")).containsExactly("core lab"); + assertThat(index.valueDisplays("host::lab_type")).containsEntry("core lab", "Core Lab"); + assertThat(index.displayName("host::lab_type")).isEqualTo("Host Lab Type"); // The previously deferred host keys are stamped onto every device of the host. Host OS defaults // to "Unknown" when the property is absent (lab2); connectivity buckets the lab status. @@ -184,13 +184,12 @@ public void build_indexesDevicesHostsAndValues() { assertThat(index.valueCount("host::host_os", "unknown")).isEqualTo(1); assertThat(index.valueCount("host::connectivity", "running")).isEqualTo(2); assertThat(index.valueCount("host::connectivity", "missing")).isEqualTo(1); - assertThat(index.valueDisplays().get("host::connectivity")).containsEntry("running", "Running"); + assertThat(index.valueDisplays("host::connectivity")).containsEntry("running", "Running"); assertThat(index.valueCount("host::daemon_status", "running")).isEqualTo(2); assertThat(index.valueCount("host::release_status", "running")).isEqualTo(2); assertThat(index.valueCount("host::release_type", "shared_lab")).isEqualTo(2); assertThat(index.valueCount("host::lab_server_version", "v42")).isEqualTo(2); - assertThat(index.displayNames()) - .containsEntry("host::daemon_status", "Host Daemon Server Status"); + assertThat(index.displayName("host::daemon_status")).isEqualTo("Host Daemon Server Status"); // Posting lists resolve the host keys through the device forward store, so filtering by a host // attribute selects the devices on matching hosts. @@ -230,7 +229,7 @@ public void build_indexesDevicesHostsAndValues() { FleetIndex hostIndex = snapshot.hostIndex(); assertThat(hostIndex.valueCount("host::device_count", "2")).isEqualTo(1); assertThat(hostIndex.valueCount("host::device_count", "1")).isEqualTo(1); - assertThat(hostIndex.displayNames()).containsEntry("host::device_count", "Device Count"); + assertThat(hostIndex.displayName("host::device_count")).isEqualTo("Device Count"); assertThat(hostIndex.valueCount("host::host_name", "lab1")).isEqualTo(1); assertThat(hostIndex.valueCount("host::host_name", "lab2")).isEqualTo(1); assertThat(hostIndex.valueCount("host::host_ip", "1.1.1.1")).isEqualTo(1); @@ -339,12 +338,11 @@ public void build_hostLabType_combinesHostPropertyAndReleaseEnum() { .inOrder(); // Indexed as lowercased terms with the original display preserved. - assertThat(index.sortedValues().get("host::lab_type")) + assertThat(index.sortedValues("host::lab_type")) .containsExactly("core lab", "fusion lab", "satellite lab", "slaas") .inOrder(); - assertThat(index.valueDisplays().get("host::lab_type")).containsEntry("slaas", "SLaaS"); - assertThat(index.valueDisplays().get("host::lab_type")) - .containsEntry("fusion lab", "Fusion Lab"); + assertThat(index.valueDisplays("host::lab_type")).containsEntry("slaas", "SLaaS"); + assertThat(index.valueDisplays("host::lab_type")).containsEntry("fusion lab", "Fusion Lab"); assertThat(index.valueCount("host::lab_type", "core lab")).isEqualTo(1); // The device on the enriched host carries the lab type; the device on the ATS-like host does @@ -392,7 +390,7 @@ public void build_atsController_termIsIdDisplayIsFriendly() { // The key is indexed and its column label is "ATS Lab". assertThat(index.keyIds()).contains("host::ats_controller"); - assertThat(index.displayNames()).containsEntry("host::ats_controller", "ATS Lab"); + assertThat(index.displayName("host::ats_controller")).isEqualTo("ATS Lab"); // The stored/filter term is the lowercased controller id, not the friendly display. LazyPostings atsPostings = new LazyPostings(atsSnapshot.devices()); @@ -402,9 +400,9 @@ public void build_atsController_termIsIdDisplayIsFriendly() { // The per-value display is the friendly name when the registry has an entry, and falls back to // the controller id when it does not. - assertThat(index.valueDisplays().get("host::ats_controller")) + assertThat(index.valueDisplays("host::ats_controller")) .containsEntry("xiaomi", "Partner Lab: Xiaomi"); - assertThat(index.valueDisplays().get("host::ats_controller")).containsEntry("acme", "acme"); + assertThat(index.valueDisplays("host::ats_controller")).containsEntry("acme", "acme"); } @Test @@ -448,17 +446,17 @@ public void build_hostAtsController_termIsIdDisplayIsFriendly() { // The key is indexed in the host index and its column label is "ATS Lab". assertThat(hostIndex.keyIds()).contains("host::ats_controller"); - assertThat(hostIndex.displayNames()).containsEntry("host::ats_controller", "ATS Lab"); + assertThat(hostIndex.displayName("host::ats_controller")).isEqualTo("ATS Lab"); // The stored/filter term is the lowercased controller id; the enriched host is selected and the // unenriched host contributes nothing. assertThat(hostIndex.valueCount("host::ats_controller", "xiaomi")).isEqualTo(1); - assertThat(hostIndex.sortedValues().get("host::ats_controller")).containsExactly("xiaomi"); + assertThat(hostIndex.sortedValues("host::ats_controller")).containsExactly("xiaomi"); LazyPostings hostPostings = LazyPostings.forHosts(snapshot.hosts()); assertThat(posting(hostPostings, "host::ats_controller", "xiaomi")).containsExactly(0); // The per-value display is the friendly name from the registry. - assertThat(hostIndex.valueDisplays().get("host::ats_controller")) + assertThat(hostIndex.valueDisplays("host::ats_controller")) .containsEntry("xiaomi", "Partner Lab: Xiaomi"); } @@ -520,8 +518,8 @@ public void emptyDimensionValue_notIndexedAsValue() { FleetIndex index = snapshot.index(); // The empty string is not a facet value: only the real value is listed for the key. - assertThat(index.sortedValues().get("dim::os")).containsExactly("android"); - assertThat(index.sortedValues().get("dim::os")).doesNotContain(""); + assertThat(index.sortedValues("dim::os")).containsExactly("android"); + assertThat(index.sortedValues("dim::os")).doesNotContain(""); // Only the real device is counted for the key, and the empty string carries no count. The // empty-value device contributes to no value. diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD index 3f9314fed4..6dbcaa6e56 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/BUILD @@ -28,13 +28,16 @@ java_library( "//src/devtools/mobileharness/api/model/proto:device_java_proto", "//src/devtools/mobileharness/api/query/proto:lab_query_java_proto", "//src/devtools/mobileharness/fe/v6/service/proto/search:search_fleet_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:dimension_overlay", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_index_builder", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_raw_data", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:fleet_snapshot", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:host_record", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/index:lazy_postings", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:dimension_overlay_raw", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/pull:fleet_data_source", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:dimension_overlay_store", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:fleet_data_refresher", "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/refresh:fleet_snapshot_store", "//src/java/com/google/devtools/mobileharness/shared/util/concurrent:thread_pools", diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStoreTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStoreTest.java new file mode 100644 index 0000000000..ff98ddee70 --- /dev/null +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/DimensionOverlayStoreTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.refresh; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.SettableFuture; +import com.google.devtools.mobileharness.fe.v6.service.proto.search.Fleet; +import com.google.devtools.mobileharness.fe.v6.service.search.index.CoreFleetIndex; +import com.google.devtools.mobileharness.fe.v6.service.search.index.DimensionOverlay; +import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetRawData; +import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.DimensionOverlayRaw; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.FleetDataSource; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link DimensionOverlayStore}. */ +@RunWith(JUnit4.class) +public final class DimensionOverlayStoreTest { + + private final ListeningExecutorService executor = newDirectExecutorService(); + private final FleetSnapshotStore snapshotStore = new FleetSnapshotStore(); + private FakeDataSource dataSource; + private DimensionOverlayStore overlayStore; + + @Before + public void setUp() { + FleetSnapshot snapshot = + FleetSnapshot.builder() + .setBuildTime(Instant.ofEpochSecond(1_000)) + .setDevices(ImmutableList.of()) + .setHosts(ImmutableList.of()) + .setIndex(CoreFleetIndex.empty()) + .setHostIndex(CoreFleetIndex.empty()) + .setUuidToIndex(ImmutableMap.of("dev-1", 0, "dev-2", 1)) + .build(); + snapshotStore.publish(Fleet.FLEET_SELF, snapshot); + + dataSource = new FakeDataSource(); + overlayStore = + new DimensionOverlayStore(ImmutableMap.of(Fleet.FLEET_SELF, dataSource), snapshotStore); + } + + @Test + public void loadOverlaysAsync_emptyKeys_returnsImmediately() throws Exception { + ImmutableMap result = + overlayStore.loadOverlaysAsync(Fleet.FLEET_SELF, ImmutableSet.of(), executor).get(); + + assertThat(result).isEmpty(); + assertThat(dataSource.pullCount.get()).isEqualTo(0); + } + + @Test + public void loadOverlaysAsync_coldKey_pullsFromDataSourceAndCaches() throws Exception { + dataSource.rawResult = + DimensionOverlayRaw.create( + "dim::carrier", ImmutableMap.of("dev-1", ImmutableList.of("Verizon"))); + + ImmutableMap result = + overlayStore + .loadOverlaysAsync(Fleet.FLEET_SELF, ImmutableSet.of("dim::carrier"), executor) + .get(); + + assertThat(result.keySet()).containsExactly("dim::carrier"); + assertThat(result.get("dim::carrier").valueCounts()).containsEntry("verizon", 1); + assertThat(dataSource.pullCount.get()).isEqualTo(1); + + // Second call hits cache; does not pull again + ImmutableMap secondResult = + overlayStore + .loadOverlaysAsync(Fleet.FLEET_SELF, ImmutableSet.of("dim::carrier"), executor) + .get(); + + assertThat(secondResult.keySet()).containsExactly("dim::carrier"); + assertThat(dataSource.pullCount.get()).isEqualTo(1); + } + + @Test + public void loadOverlaysAsync_concurrentCallsForSameKey_deduplicates() throws Exception { + SettableFuture inFlight = SettableFuture.create(); + dataSource.setPendingFuture(inFlight); + + ListenableFuture> future1 = + overlayStore.loadOverlaysAsync(Fleet.FLEET_SELF, ImmutableSet.of("dim::carrier"), executor); + ListenableFuture> future2 = + overlayStore.loadOverlaysAsync(Fleet.FLEET_SELF, ImmutableSet.of("dim::carrier"), executor); + + assertThat(dataSource.pullCount.get()).isEqualTo(1); + + // Complete the in-flight pull + inFlight.set( + DimensionOverlayRaw.create( + "dim::carrier", ImmutableMap.of("dev-1", ImmutableList.of("Verizon")))); + + assertThat(future1.get()).containsKey("dim::carrier"); + assertThat(future2.get()).containsKey("dim::carrier"); + assertThat(dataSource.pullCount.get()).isEqualTo(1); + } + + private static final class FakeDataSource implements FleetDataSource { + final AtomicInteger pullCount = new AtomicInteger(0); + volatile DimensionOverlayRaw rawResult = + DimensionOverlayRaw.create("dim::empty", ImmutableMap.of()); + volatile SettableFuture pendingFuture = null; + + void setPendingFuture(SettableFuture pending) { + this.pendingFuture = pending; + } + + @Override + public Fleet fleet() { + return Fleet.FLEET_SELF; + } + + @Override + public ListenableFuture pull() { + throw new UnsupportedOperationException(); + } + + @Override + public ListenableFuture pullDimension(String keyId) { + pullCount.incrementAndGet(); + if (pendingFuture != null) { + return pendingFuture; + } + return immediateFuture(rawResult); + } + } +} diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetDataRefresherTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetDataRefresherTest.java index d4b4aa8705..06ea1836b1 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetDataRefresherTest.java +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetDataRefresherTest.java @@ -32,6 +32,7 @@ import com.google.devtools.mobileharness.fe.v6.service.proto.search.Fleet; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndexBuilder; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetRawData; +import com.google.devtools.mobileharness.fe.v6.service.search.pull.DimensionOverlayRaw; import com.google.devtools.mobileharness.fe.v6.service.search.pull.FleetDataSource; import com.google.devtools.mobileharness.shared.util.concurrent.ThreadPools; import com.google.inject.Guice; @@ -150,5 +151,10 @@ public Fleet fleet() { public ListenableFuture pull() { return result; } + + @Override + public ListenableFuture pullDimension(String keyId) { + return immediateFuture(DimensionOverlayRaw.create(keyId, ImmutableMap.of())); + } } } diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetSnapshotStoreTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetSnapshotStoreTest.java index af1a8cf6b3..87f0041fa6 100644 --- a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetSnapshotStoreTest.java +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/refresh/FleetSnapshotStoreTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.mobileharness.fe.v6.service.proto.search.Fleet; -import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetIndex; +import com.google.devtools.mobileharness.fe.v6.service.search.index.CoreFleetIndex; import com.google.devtools.mobileharness.fe.v6.service.search.index.FleetSnapshot; import com.google.devtools.mobileharness.fe.v6.service.search.index.HostRecord; import com.google.devtools.mobileharness.fe.v6.service.search.index.LazyPostings; @@ -81,8 +81,8 @@ public void hostPostings_resolveHostKeysOverPublishedHosts() { .setBuildTime(Instant.ofEpochSecond(1_700_000_000L)) .setDevices(ImmutableList.of()) .setHosts(ImmutableList.of(host("lab1", 2), host("lab2", 1))) - .setIndex(FleetIndex.empty()) - .setHostIndex(FleetIndex.empty()) + .setIndex(CoreFleetIndex.empty()) + .setHostIndex(CoreFleetIndex.empty()) .build(); store.publish(Fleet.FLEET_SELF, snapshot); @@ -100,8 +100,8 @@ private static FleetSnapshot snapshotAt(long epochSecond) { .setBuildTime(Instant.ofEpochSecond(epochSecond)) .setDevices(ImmutableList.of()) .setHosts(ImmutableList.of()) - .setIndex(FleetIndex.empty()) - .setHostIndex(FleetIndex.empty()) + .setIndex(CoreFleetIndex.empty()) + .setHostIndex(CoreFleetIndex.empty()) .build(); } diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/BUILD b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/BUILD new file mode 100644 index 0000000000..697a2a4bb7 --- /dev/null +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/BUILD @@ -0,0 +1,47 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_java//java:java_library.bzl", "java_library") +load("//src/javatests/com/google/devtools/mobileharness/builddefs:junit_test_suites.bzl", "junit_test_suites") + +package( + default_applicable_licenses = ["//:license"], +) + +java_library( + name = "tests", + testonly = 1, + srcs = glob(["*.java"]), + deps = [ + "//src/devtools/mobileharness/api/model/proto:device_java_proto", + "//src/devtools/mobileharness/api/query/proto:lab_query_java_proto", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema:device_info_source", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema:device_keys", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema:key_descriptor", + "//src/java/com/google/devtools/mobileharness/fe/v6/service/search/schema:key_registry", + "//src/javatests/com/google/devtools/mobileharness/builddefs:truth", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:junit_junit", + "@protobuf//:protobuf_java", + "@protobuf//:protobuf_java_util", + ], +) + +junit_test_suites( + name = "gen_tests", + sizes = ["small"], + deps = [":tests"], +) diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyRegistryTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyRegistryTest.java new file mode 100644 index 0000000000..3e58f3a44d --- /dev/null +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/DeviceKeyRegistryTest.java @@ -0,0 +1,184 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.mobileharness.api.model.proto.Device.DeviceCompositeDimension; +import com.google.devtools.mobileharness.api.model.proto.Device.DeviceDimension; +import com.google.devtools.mobileharness.api.model.proto.Device.DeviceFeature; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.DeviceInfo; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabInfo; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQuery.Mask.DeviceInfoMask; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQuery.Mask.LabInfoMask; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.FieldMask; +import com.google.protobuf.util.FieldMaskUtil; +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class DeviceKeyRegistryTest { + + private final AtsDeviceKeyRegistry registry = new AtsDeviceKeyRegistry(); + + @Test + public void containsCommonDeviceKeysAndProjectedCommonHostKeys() { + assertThat(registry.getKey("device_field::uuid")).isPresent(); + assertThat(registry.getKey("dimension::model")).isPresent(); + assertThat(registry.getKey("dimension::os")).isPresent(); + assertThat(registry.getKey("dimension::device_class_name")).isPresent(); + assertThat(registry.getKey("dimension::manufacturer")).isPresent(); + // Standalone ATS exclusive key. + assertThat(registry.getKey("device_config::wifi_ssid")).isPresent(); + // Projected common host keys are usable in device search. + assertThat(registry.getKey("host_field::host_name")).isPresent(); + assertThat(registry.getKey("host_field::connectivity")).isPresent(); + assertThat(registry.getKey("host_property::host_os")).isPresent(); + } + + @Test + public void excludesOnePkeysAndUnprojectedHostKeys() { + // 1P device-native keys (owner/executor/quarantine/pool/lab_location) are not in OSS. + assertThat(registry.getKey("device_field::owner")).isEmpty(); + assertThat(registry.getKey("device_field::executor")).isEmpty(); + assertThat(registry.getKey("device_field::quarantined")).isEmpty(); + assertThat(registry.getKey("dimension::pool")).isPresent(); // long-tail mint, not built-in + assertThat(registry.getKey("dimension::pool").get().isLongTail()).isTrue(); + // device_count is host-search only: never projected into device search. + assertThat(registry.getKey("host_field::device_count")).isEmpty(); + // 1P host keys are not projected in OSS. + assertThat(registry.getKey("host_field::lab_type")).isEmpty(); + } + + @Test + public void builtInKeysAreNotLongTail() { + assertThat(registry.getKey("device_field::uuid").get().isLongTail()).isFalse(); + assertThat(registry.getKey("dimension::model").get().isLongTail()).isFalse(); + assertThat(registry.getKey("host_field::host_name").get().isLongTail()).isFalse(); + } + + @Test + public void mintsLongTailDimensionAndHostProperty() { + Optional dim = registry.getKey("dimension::carrier"); + assertThat(dim).isPresent(); + assertThat(dim.get().isLongTail()).isTrue(); + assertThat(dim.get().display().name()).isEqualTo("carrier"); + + Optional prop = registry.getKey("host_property::rack"); + assertThat(prop).isPresent(); + assertThat(prop.get().isLongTail()).isTrue(); + assertThat(prop.get().display().name()).isEqualTo("rack"); + + assertThat(registry.dimensionKey("carrier").isLongTail()).isTrue(); + assertThat(registry.hostPropertyKey("rack").isLongTail()).isTrue(); + // An unknown non-mintable namespace is not a key. + assertThat(registry.getKey("device_field::bogus")).isEmpty(); + } + + @Test + public void displayNamesUseDeviceSearchLabels() { + assertThat(registry.displayName("device_field::uuid")).hasValue("UUID"); + assertThat(registry.displayName("dimension::model")).hasValue("Model"); + // A projected host key shows its device-search name. + assertThat(registry.displayName("host_field::connectivity")) + .hasValue("Host Lab Server Connectivity"); + } + + @Test + public void deriveDeviceInfoMask_unionsCommonSources_excludesOnePAndQuarantine() { + DeviceInfoMask mask = registry.deriveDeviceInfoMask(); + assertThat(mask.getFieldMask().getPathsList()) + .containsAtLeast( + "device_locator.id", + "device_status", + "device_feature.type", + "device_feature.driver", + "device_feature.decorator", + "device_feature.composite_dimension"); + assertThat(mask.getSupportedDimensionsMask().getDimensionNamesList()) + .containsAtLeast( + "model", + "os", + "sdk_version", + "software_version", + "device_form", + "device_class_name", + "manufacturer"); + // Quarantine is 1P-only, so OSS does not pull device_condition and never names "quarantined". + assertThat(mask.getFieldMask().getPathsList()).doesNotContain("device_condition"); + assertThat(mask.getSupportedDimensionsMask().getDimensionNamesList()) + .doesNotContain("quarantined"); + // Host paths belong to the LabInfoMask. + assertThat(mask.getFieldMask().getPathsList()).doesNotContain("lab_locator.host_name"); + assertMaskPathsExist(DeviceInfo.getDescriptor(), mask.getFieldMask()); + } + + @Test + public void deriveLabInfoMask_fromProjectedHostKeys() { + LabInfoMask mask = registry.deriveLabInfoMask(); + assertThat(mask.getFieldMask().getPathsList()) + .containsAtLeast( + "lab_locator.host_name", + "lab_locator.ip", + "lab_status", + "lab_server_feature.host_properties"); + assertThat(mask.getFieldMask().getPathsList()).doesNotContain("device_locator.id"); + assertMaskPathsExist(LabInfo.getDescriptor(), mask.getFieldMask()); + } + + @Test + public void deviceInfoSource_extractsValueFromProto() { + DeviceInfo deviceInfo = + DeviceInfo.newBuilder() + .setDeviceFeature( + DeviceFeature.newBuilder() + .setCompositeDimension( + DeviceCompositeDimension.newBuilder() + .addSupportedDimension( + DeviceDimension.newBuilder().setName("model").setValue("Pixel 8")))) + .build(); + DeviceInfoSource modelSource = DeviceKeys.MODEL.deviceInfoSources().get(0); + assertThat(modelSource.extract(deviceInfo)).containsExactly("Pixel 8"); + } + + @Test + public void constructor_rejectsDuplicateId() { + DeviceKeyDescriptor dup = + DeviceKeyDescriptor.builder() + .setId("device_field::uuid") + .setDeviceInfoSource(DeviceInfoSource.dimension("uuid")) + .setDisplay(KeyDisplay.of("Dup")) + .build(); + assertThrows( + IllegalArgumentException.class, () -> new DeviceKeyRegistry(ImmutableList.of(dup)) {}); + } + + private static void assertMaskPathsExist(Descriptor descriptor, FieldMask mask) { + for (String path : mask.getPathsList()) { + assertWithMessage("%s is not a field path of %s", path, descriptor.getFullName()) + .that(FieldMaskUtil.isValid(descriptor, path)) + .isTrue(); + } + assertThat(mask.getPathsList()).isNotEmpty(); + } +} diff --git a/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyRegistryTest.java b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyRegistryTest.java new file mode 100644 index 0000000000..626812a3a1 --- /dev/null +++ b/src/javatests/com/google/devtools/mobileharness/fe/v6/service/search/schema/HostKeyRegistryTest.java @@ -0,0 +1,101 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.mobileharness.fe.v6.service.search.schema; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabInfo; +import com.google.devtools.mobileharness.api.query.proto.LabQueryProto.LabQuery.Mask.LabInfoMask; +import com.google.protobuf.util.FieldMaskUtil; +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class HostKeyRegistryTest { + + private final AtsHostKeyRegistry registry = new AtsHostKeyRegistry(); + + @Test + public void containsCommonHostKeys() { + assertThat(registry.getKey("host_field::host_name")).isPresent(); + assertThat(registry.getKey("host_field::host_ip")).isPresent(); + assertThat(registry.getKey("host_field::connectivity")).isPresent(); + assertThat(registry.getKey("host_property::host_os")).isPresent(); + assertThat(registry.getKey("host_field::lab_server_version")).isPresent(); + assertThat(registry.getKey("host_field::device_count")).isPresent(); + } + + @Test + public void excludesOnePAndPartnerKeys() { + assertThat(registry.getKey("host_field::release_status")).isEmpty(); + assertThat(registry.getKey("host_field::lab_type")).isEmpty(); + assertThat(registry.getKey("host_field::ats_controller")).isEmpty(); + // A device key is not a host key. + assertThat(registry.getKey("device_field::uuid")).isEmpty(); + } + + @Test + public void displayNamesUseHostSearchLabels() { + assertThat(registry.displayName("host_field::host_name")).hasValue("Host Name"); + // Host search shows the bare label, not the "Host ..." device-search form. + assertThat(registry.displayName("host_field::connectivity")) + .hasValue("Lab Server Connectivity"); + } + + @Test + public void mintsLongTailHostProperty() { + Optional prop = registry.getKey("host_property::rack"); + assertThat(prop).isPresent(); + assertThat(prop.get().isLongTail()).isTrue(); + assertThat(prop.get().display().name()).isEqualTo("rack"); + assertThat(registry.hostPropertyKey("rack").isLongTail()).isTrue(); + // A host_field:: id that is not built in cannot be minted. + assertThat(registry.getKey("host_field::bogus")).isEmpty(); + } + + @Test + public void deriveLabInfoMask_unionsHostSources() { + LabInfoMask mask = registry.deriveLabInfoMask(); + assertThat(mask.getFieldMask().getPathsList()) + .containsAtLeast( + "lab_locator.host_name", + "lab_locator.ip", + "lab_status", + "lab_server_feature.host_properties"); + for (String path : mask.getFieldMask().getPathsList()) { + assertWithMessage("%s is not a field path of LabInfo", path) + .that(FieldMaskUtil.isValid(LabInfo.getDescriptor(), path)) + .isTrue(); + } + } + + @Test + public void constructor_rejectsDevicePrefixedId() { + HostKeyDescriptor bad = + HostKeyDescriptor.builder() + .setId("device_field::bogus") + .setDisplay(KeyDisplay.of("Bogus")) + .build(); + assertThrows( + IllegalArgumentException.class, () -> new HostKeyRegistry(ImmutableList.of(bad)) {}); + } +}