diff --git a/lore-proto/proto/lore/repository/v1/repository.proto b/lore-proto/proto/lore/repository/v1/repository.proto index 3e08253dd..07cd422de 100644 --- a/lore-proto/proto/lore/repository/v1/repository.proto +++ b/lore-proto/proto/lore/repository/v1/repository.proto @@ -23,6 +23,12 @@ service RepositoryService { rpc RepositoryGet (RepositoryGetRequest) returns (RepositoryGetResponse); // Stream all repositories the caller is authorised to see. rpc RepositoryList (RepositoryListRequest) returns (stream RepositoryListResponse); + // Count the repositories the caller is authorised to see, without + // enumerating them. Applies the same optional filters as + // `RepositoryList`. Cheaper than `RepositoryList` because no per-repo + // metadata is streamed back to the caller; when no filter is set, the + // server can answer from the repository id set alone. + rpc RepositoryCount (RepositoryCountRequest) returns (RepositoryCountResponse); // Cheap hash-only read of a repository's current metadata pointer. rpc RepositoryMetadataGet(RepositoryMetadataGetRequest) returns (RepositoryMetadataGetResponse); // Compare-and-swap update of a repository's metadata pointer. CAS miss is @@ -118,6 +124,23 @@ message RepositoryListResponse { lore.model.v1.Repository repository = 1; } +// Request to count repositories the caller is authorised to see. Filter +// semantics mirror `RepositoryListRequest`; a repository is counted iff +// `RepositoryList` with the same filters would emit it. +message RepositoryCountRequest { + // If set, only repositories whose `creator` field is an exact + // byte-for-byte case-sensitive match are counted. + optional string creator = 1; +} + +// Response carrying the count of matching repositories. `count` defaults +// to zero, so an absent or unset value is safe to read as zero. +message RepositoryCountResponse { + // Number of repositories matching the request's filters and the + // caller's authorisation. + uint64 count = 1; +} + // Request for a cheap hash-only read of a repository's metadata pointer. message RepositoryMetadataGetRequest { // Repository id. diff --git a/lore-proto/src/grpc/lore.repository.v1.rs b/lore-proto/src/grpc/lore.repository.v1.rs index 91ea4c223..b33fc0fba 100644 --- a/lore-proto/src/grpc/lore.repository.v1.rs +++ b/lore-proto/src/grpc/lore.repository.v1.rs @@ -181,6 +181,45 @@ impl ::prost::Name for RepositoryListResponse { "/lore.repository.v1.RepositoryListResponse".into() } } +/// Request to count repositories the caller is authorised to see. Filter +/// semantics mirror `RepositoryListRequest`; a repository is counted iff +/// `RepositoryList` with the same filters would emit it. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RepositoryCountRequest { + /// If set, only repositories whose `creator` field is an exact + /// byte-for-byte case-sensitive match are counted. + #[prost(string, optional, tag = "1")] + pub creator: ::core::option::Option<::prost::alloc::string::String>, +} +impl ::prost::Name for RepositoryCountRequest { + const NAME: &'static str = "RepositoryCountRequest"; + const PACKAGE: &'static str = "lore.repository.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.repository.v1.RepositoryCountRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.repository.v1.RepositoryCountRequest".into() + } +} +/// Response carrying the count of matching repositories. `count` defaults +/// to zero, so an absent or unset value is safe to read as zero. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RepositoryCountResponse { + /// Number of repositories matching the request's filters and the + /// caller's authorisation. + #[prost(uint64, tag = "1")] + pub count: u64, +} +impl ::prost::Name for RepositoryCountResponse { + const NAME: &'static str = "RepositoryCountResponse"; + const PACKAGE: &'static str = "lore.repository.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.repository.v1.RepositoryCountResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.repository.v1.RepositoryCountResponse".into() + } +} /// Request for a cheap hash-only read of a repository's metadata pointer. #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RepositoryMetadataGetRequest { @@ -484,6 +523,40 @@ pub mod repository_service_client { ); self.inner.server_streaming(req, path, codec).await } + /// Count the repositories the caller is authorised to see, without + /// enumerating them. Applies the same optional filters as + /// `RepositoryList`. Cheaper than `RepositoryList` because no per-repo + /// metadata is streamed back to the caller; when no filter is set, the + /// server can answer from the repository id set alone. + pub async fn repository_count( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/lore.repository.v1.RepositoryService/RepositoryCount", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "lore.repository.v1.RepositoryService", + "RepositoryCount", + ), + ); + self.inner.unary(req, path, codec).await + } /// Cheap hash-only read of a repository's current metadata pointer. pub async fn repository_metadata_get( &mut self, @@ -603,6 +676,18 @@ pub mod repository_service_server { tonic::Response, tonic::Status, >; + /// Count the repositories the caller is authorised to see, without + /// enumerating them. Applies the same optional filters as + /// `RepositoryList`. Cheaper than `RepositoryList` because no per-repo + /// metadata is streamed back to the caller; when no filter is set, the + /// server can answer from the repository id set alone. + async fn repository_count( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; /// Cheap hash-only read of a repository's current metadata pointer. async fn repository_metadata_get( &self, @@ -887,6 +972,52 @@ pub mod repository_service_server { }; Box::pin(fut) } + "/lore.repository.v1.RepositoryService/RepositoryCount" => { + #[allow(non_camel_case_types)] + struct RepositoryCountSvc(pub Arc); + impl< + T: RepositoryService, + > tonic::server::UnaryService + for RepositoryCountSvc { + type Response = super::RepositoryCountResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::repository_count(&inner, request) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RepositoryCountSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/lore.repository.v1.RepositoryService/RepositoryMetadataGet" => { #[allow(non_camel_case_types)] struct RepositoryMetadataGetSvc(pub Arc); diff --git a/lore-proto/tests/v1_repository.rs b/lore-proto/tests/v1_repository.rs index 33e0fe708..f4fedb3e0 100644 --- a/lore-proto/tests/v1_repository.rs +++ b/lore-proto/tests/v1_repository.rs @@ -1,8 +1,10 @@ // SPDX-FileCopyrightText: 2026 Epic Games, Inc. // SPDX-License-Identifier: MIT -//! Smoke test verifying `lore.repository.v1` carries the 6 RPCs' request / +//! Smoke test verifying `lore.repository.v1` carries the 7 RPCs' request / //! response messages. +use lore_proto::lore::repository::v1::RepositoryCountRequest; +use lore_proto::lore::repository::v1::RepositoryCountResponse; use lore_proto::lore::repository::v1::RepositoryCreateRequest; use lore_proto::lore::repository::v1::RepositoryCreateResponse; use lore_proto::lore::repository::v1::RepositoryDeleteRequest; @@ -27,6 +29,8 @@ fn v1_repository_request_response_types_default() { let _ = RepositoryGetResponse::default(); let _ = RepositoryListRequest::default(); let _ = RepositoryListResponse::default(); + let _ = RepositoryCountRequest::default(); + let _ = RepositoryCountResponse::default(); let _ = RepositoryMetadataGetRequest::default(); let _ = RepositoryMetadataGetResponse::default(); let _ = RepositoryMetadataSetRequest::default(); @@ -60,6 +64,9 @@ fn v1_repository_field_shapes() { let RepositoryListRequest { creator: _ } = RepositoryListRequest::default(); let RepositoryListResponse { repository: _ } = RepositoryListResponse::default(); + let RepositoryCountRequest { creator: _ } = RepositoryCountRequest::default(); + let RepositoryCountResponse { count: _ } = RepositoryCountResponse::default(); + let RepositoryMetadataGetRequest { id: _ } = RepositoryMetadataGetRequest::default(); let RepositoryMetadataGetResponse { metadata: _ } = RepositoryMetadataGetResponse::default(); diff --git a/lore-server/src/grpc/repository/v1/mod.rs b/lore-server/src/grpc/repository/v1/mod.rs index 3b9497a68..46a4e7018 100644 --- a/lore-server/src/grpc/repository/v1/mod.rs +++ b/lore-server/src/grpc/repository/v1/mod.rs @@ -1,5 +1,6 @@ // SPDX-FileCopyrightText: 2026 Epic Games, Inc. // SPDX-License-Identifier: MIT +pub mod repository_count; pub mod repository_create; pub mod repository_delete; pub mod repository_get; diff --git a/lore-server/src/grpc/repository/v1/repository_count.rs b/lore-server/src/grpc/repository/v1/repository_count.rs new file mode 100644 index 000000000..3f4b9ff3c --- /dev/null +++ b/lore-server/src/grpc/repository/v1/repository_count.rs @@ -0,0 +1,335 @@ +// SPDX-FileCopyrightText: 2026 LoreLab.io +// SPDX-License-Identifier: MIT +use std::sync::Arc; + +use lore_base::lore_spawn; +use lore_base::runtime::LORE_CONTEXT; +use lore_base::types::Context; +use lore_proto::lore::repository::v1::RepositoryCountRequest; +use lore_proto::lore::repository::v1::RepositoryCountResponse; +use lore_revision::lore::RepositoryId; +use lore_revision::lore::execution_context; +use lore_revision::repository; +use lore_revision::repository::RepositoryContext; +use tokio::task::JoinSet; +use tokio_stream::StreamExt; +use tonic::Request; +use tonic::Response; +use tonic::Status; +use tracing::Instrument; +use tracing::debug; + +use crate::grpc::ServerResultExt; +use crate::grpc::extract_authorization_header; +use crate::grpc::extract_correlation_id; +use crate::grpc::get_user_id; +use crate::grpc::handlers::repository_list::lookup_authorized_repositories; +use crate::util::setup_execution; + +/// `lore.repository.v1.RepositoryService.RepositoryCount` handler. +/// +/// Returns the number of repositories the caller is authorised to see, +/// applying the same filter semantics as `RepositoryList`. When no +/// filter is set the count is answered directly from the id set with no +/// per-repository metadata reads; a `creator` filter forces per-repo +/// metadata loads so the exact same predicate can be evaluated. +#[tracing::instrument(name = "RepositoryCount::v1::handle", skip_all)] +pub async fn handler( + request: Request, + auth_url: Option, + immutable_store: Arc, + mutable_store: Arc, +) -> Result, Status> { + let user_id = get_user_id(request.extensions()); + let correlation_id = extract_correlation_id(&request).unwrap_or_default(); + let authorization = extract_authorization_header(&request); + let req = request.into_inner(); + let creator_filter = req.creator; + + let execution = setup_execution(module_path!(), correlation_id, user_id); + + LORE_CONTEXT + .scope(execution.clone(), async move { + let candidate_ids = candidate_ids( + immutable_store.clone(), + mutable_store.clone(), + auth_url, + authorization, + ) + .await?; + + debug!(count = candidate_ids.len(), "Repository count candidates"); + + let count = if let Some(filter) = creator_filter { + count_matching_creator(immutable_store, mutable_store, candidate_ids, filter).await + } else { + candidate_ids.len() as u64 + }; + + Ok(Response::new(RepositoryCountResponse { count })) + }) + .await +} + +/// Resolve the caller's authorised repository id set. When an auth URL is +/// configured the ids come from the auth service; otherwise every +/// locally-known repository id is returned. Mirrors the same-named helper +/// inside `repository_list.rs` (separated to avoid widening visibility). +async fn candidate_ids( + immutable_store: Arc, + mutable_store: Arc, + auth_url: Option, + authorization: Option, +) -> Result, Status> { + if let Some(auth_url) = auth_url { + let ids = lookup_authorized_repositories(auth_url, authorization).await?; + Ok(ids.into_iter().map(RepositoryId::from).collect()) + } else { + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store, + mutable_store, + Context::default().into(), + )); + let mut stream = repository::list_local(repository) + .await + .warn_map_err(|err| Status::internal(format!("Failed to list repositories: {err}")))?; + let mut out = Vec::new(); + while let Some(id) = stream.next().await { + out.push(id.into()); + } + Ok(out) + } +} + +/// Fan out per-repo metadata loads under a `JoinSet`, counting only those +/// whose `creator` matches `filter`. Per-repo load failures are logged +/// and skipped, matching `RepositoryList`'s tolerance for missing or +/// corrupt metadata blobs. +async fn count_matching_creator( + immutable_store: Arc, + mutable_store: Arc, + candidate_ids: Vec, + filter: String, +) -> u64 { + let mut tasks: JoinSet = JoinSet::new(); + for id in candidate_ids { + let immutable_store = immutable_store.clone(); + let mutable_store = mutable_store.clone(); + let filter = filter.clone(); + lore_spawn!( + tasks, + LORE_CONTEXT + .scope(execution_context(), async move { + creator_matches(immutable_store, mutable_store, id, &filter).await + }) + .in_current_span(), + ); + } + + let mut count: u64 = 0; + while let Some(result) = tasks.join_next().await { + match result { + Ok(true) => count += 1, + Ok(false) => {} + Err(err) => debug!(%err, "Repository count: metadata task panicked, skipping"), + } + } + count +} + +async fn creator_matches( + immutable_store: Arc, + mutable_store: Arc, + id: RepositoryId, + filter: &str, +) -> bool { + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store, + mutable_store, + id, + )); + + let metadata_hash = match repository::metadata_hash(repository.clone()).await { + Ok(hash) => hash, + Err(err) => { + debug!(%id, %err, "Repository count: metadata hash unavailable, skipping"); + return false; + } + }; + let metadata = match repository::metadata(repository, metadata_hash).await { + Ok(metadata) => metadata, + Err(err) => { + debug!(%id, %err, "Repository count: metadata blob unavailable, skipping"); + return false; + } + }; + + metadata.creator.as_str() == filter +} + +#[cfg(test)] +mod tests { + use lore_revision::repository::RepositoryMetadata; + + use super::*; + use crate::store::test_store_create; + + async fn seed_repository( + immutable: Arc, + mutable: Arc, + id_byte: u8, + creator: &str, + ) { + let id_bytes = [id_byte; 16]; + let name = format!("repo-{id_byte}"); + let repo_id = RepositoryId::from(Context::from(id_bytes)); + let repo_ctx = Arc::new(RepositoryContext::new_server_context( + immutable, mutable, repo_id, + )); + let hash = repository::metadata_store( + repo_ctx.clone(), + RepositoryMetadata { + name: name.clone(), + creator: creator.to_string(), + ..Default::default() + }, + ) + .await + .unwrap(); + repository::metadata_store_hash(repo_ctx.clone(), hash) + .await + .unwrap(); + // Register the id in the local repository index so `list_local` + // returns it - otherwise the handler's unfiltered count sees zero + // even after metadata is written. + repository::store_name_to_id(repo_ctx, name, repo_id) + .await + .unwrap(); + } + + #[tokio::test] + async fn empty_store_counts_zero() { + let (immutable, mutable, execution) = test_store_create().await.unwrap(); + LORE_CONTEXT + .scope(execution, async move { + let response = handler( + Request::new(RepositoryCountRequest { creator: None }), + None, + immutable, + mutable, + ) + .await + .unwrap(); + assert_eq!(response.into_inner().count, 0); + }) + .await; + } + + #[tokio::test] + async fn unfiltered_returns_total_count() { + let (immutable, mutable, execution) = test_store_create().await.unwrap(); + LORE_CONTEXT + .scope(execution, async move { + seed_repository(immutable.clone(), mutable.clone(), 1, "alice").await; + seed_repository(immutable.clone(), mutable.clone(), 2, "bob").await; + seed_repository(immutable.clone(), mutable.clone(), 3, "alice").await; + + let response = handler( + Request::new(RepositoryCountRequest { creator: None }), + None, + immutable, + mutable, + ) + .await + .unwrap(); + assert_eq!(response.into_inner().count, 3); + }) + .await; + } + + #[tokio::test] + async fn creator_filter_counts_matches_only() { + let (immutable, mutable, execution) = test_store_create().await.unwrap(); + LORE_CONTEXT + .scope(execution, async move { + seed_repository(immutable.clone(), mutable.clone(), 1, "alice").await; + seed_repository(immutable.clone(), mutable.clone(), 2, "bob").await; + seed_repository(immutable.clone(), mutable.clone(), 3, "alice").await; + + let response = handler( + Request::new(RepositoryCountRequest { + creator: Some("alice".to_string()), + }), + None, + immutable, + mutable, + ) + .await + .unwrap(); + assert_eq!(response.into_inner().count, 2); + }) + .await; + } + + #[tokio::test] + async fn creator_filter_with_no_matches_counts_zero() { + let (immutable, mutable, execution) = test_store_create().await.unwrap(); + LORE_CONTEXT + .scope(execution, async move { + seed_repository(immutable.clone(), mutable.clone(), 1, "alice").await; + seed_repository(immutable.clone(), mutable.clone(), 2, "bob").await; + + let response = handler( + Request::new(RepositoryCountRequest { + creator: Some("carol".to_string()), + }), + None, + immutable, + mutable, + ) + .await + .unwrap(); + assert_eq!(response.into_inner().count, 0); + }) + .await; + } + + #[tokio::test] + async fn corrupt_metadata_is_skipped_under_filter() { + // Two healthy repos plus one whose id is registered locally but + // whose metadata is missing - the count must reflect only the + // matching healthy repos, not error out. + let (immutable, mutable, execution) = test_store_create().await.unwrap(); + LORE_CONTEXT + .scope(execution, async move { + seed_repository(immutable.clone(), mutable.clone(), 1, "alice").await; + seed_repository(immutable.clone(), mutable.clone(), 2, "alice").await; + + // Register a bare id in the repository index without + // seeding any metadata for it - `metadata_hash` will + // fail, exercising the skip-on-error path. + let bare_id = RepositoryId::from(Context::from([9u8; 16])); + let bare_ctx = Arc::new(RepositoryContext::new_server_context( + immutable.clone(), + mutable.clone(), + bare_id, + )); + repository::store_name_to_id(bare_ctx, "bare-repo", bare_id) + .await + .unwrap(); + + let response = handler( + Request::new(RepositoryCountRequest { + creator: Some("alice".to_string()), + }), + None, + immutable, + mutable, + ) + .await + .unwrap(); + assert_eq!(response.into_inner().count, 2); + }) + .await; + } +} diff --git a/lore-server/src/grpc/repository/v1/service.rs b/lore-server/src/grpc/repository/v1/service.rs index b1af655e7..be5915290 100644 --- a/lore-server/src/grpc/repository/v1/service.rs +++ b/lore-server/src/grpc/repository/v1/service.rs @@ -4,6 +4,8 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; +use lore_proto::lore::repository::v1::RepositoryCountRequest; +use lore_proto::lore::repository::v1::RepositoryCountResponse; use lore_proto::lore::repository::v1::RepositoryCreateRequest; use lore_proto::lore::repository::v1::RepositoryCreateResponse; use lore_proto::lore::repository::v1::RepositoryDeleteRequest; @@ -24,6 +26,7 @@ use tonic::Response; use tonic::Status; use tonic::codegen::tokio_stream::Stream; +use super::repository_count; use super::repository_create; use super::repository_delete; use super::repository_get; @@ -163,6 +166,22 @@ impl RepositoryService for LoreRepositoryV1Service { .await } + async fn repository_count( + &self, + request: Request, + ) -> Result, Status> { + timeout_grpc( + self.rpc_timeout, + repository_count::handler( + request, + self.auth_url(), + self.immutable_store.clone(), + self.mutable_store.clone(), + ), + ) + .await + } + async fn repository_metadata_get( &self, request: Request,