diff --git a/AGENTS.md b/AGENTS.md index e6c99d2..9c6d5dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,7 @@ control plane, or runtime AI filter. - `src/Bower.Ocsf`: OCSF normalisation engine and source mappers. - `src/Bower.Detection`: Sigma-compatible detection rules engine. - `src/Bower.Pipeline`: declarative telemetry pipeline model, templates and validation. +- `src/Bower.Analytics`: telemetry quality and coverage scoring. - `schemas`, `policies`, `deploy`, `docs`, `tests`: versioned product assets. Inspect nearest `AGENTS.md` before editing. diff --git a/Bower.sln b/Bower.sln index dd8c82b..feb8ae4 100644 --- a/Bower.sln +++ b/Bower.sln @@ -43,6 +43,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Detection", "src\Bowe EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Pipeline", "src\Bower.Pipeline\Bower.Pipeline.csproj", "{E125F241-5F0D-43FD-8781-092995E1D309}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Analytics", "src\Bower.Analytics\Bower.Analytics.csproj", "{F1BFA9A3-4762-47C4-B97D-578A105A8D6C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -281,6 +283,18 @@ Global {E125F241-5F0D-43FD-8781-092995E1D309}.Release|x64.Build.0 = Release|Any CPU {E125F241-5F0D-43FD-8781-092995E1D309}.Release|x86.ActiveCfg = Release|Any CPU {E125F241-5F0D-43FD-8781-092995E1D309}.Release|x86.Build.0 = Release|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Debug|x64.ActiveCfg = Debug|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Debug|x64.Build.0 = Debug|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Debug|x86.ActiveCfg = Debug|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Debug|x86.Build.0 = Debug|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Release|Any CPU.Build.0 = Release|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Release|x64.ActiveCfg = Release|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Release|x64.Build.0 = Release|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Release|x86.ActiveCfg = Release|Any CPU + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -292,5 +306,6 @@ Global {F95A8CF7-7C6D-49DF-853D-F845DB65C26F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {C63ADD85-CA8A-49DC-9366-30E9426C7062} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {E125F241-5F0D-43FD-8781-092995E1D309} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {F1BFA9A3-4762-47C4-B97D-578A105A8D6C} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} EndGlobalSection EndGlobal diff --git a/src/Bower.Analytics/AGENTS.md b/src/Bower.Analytics/AGENTS.md new file mode 100644 index 0000000..422a344 --- /dev/null +++ b/src/Bower.Analytics/AGENTS.md @@ -0,0 +1,4 @@ +# Analytics instructions + +Quality scoring is deterministic and explainable. Never invent coverage for +unobserved sources. Scores must include component breakdowns and evidence counts. diff --git a/src/Bower.Analytics/Bower.Analytics.csproj b/src/Bower.Analytics/Bower.Analytics.csproj new file mode 100644 index 0000000..32916dd --- /dev/null +++ b/src/Bower.Analytics/Bower.Analytics.csproj @@ -0,0 +1,5 @@ + + + + + diff --git a/src/Bower.Analytics/TelemetryQualityAssessor.cs b/src/Bower.Analytics/TelemetryQualityAssessor.cs new file mode 100644 index 0000000..9628478 --- /dev/null +++ b/src/Bower.Analytics/TelemetryQualityAssessor.cs @@ -0,0 +1,249 @@ +using Bower.Contracts; + +namespace Bower.Analytics; + +public sealed record SourceCoverageObservation( + string SourceId, + string SourceType, + bool Healthy, + DateTimeOffset? LastEventAt, + long EventsObserved, + bool Required); + +public sealed record FieldCompletenessObservation( + string FieldName, + long PresentCount, + long TotalCount, + bool Required); + +public sealed record QualityAssessmentInput( + IReadOnlyList Sources, + IReadOnlyList Fields, + IReadOnlyList? SampleEvents = null, + TimeSpan? FreshnessWindow = null); + +public sealed record QualityComponentScore( + string Name, + int Score, + int Weight, + string Summary, + IReadOnlyList Evidence); + +public sealed record TelemetryQualityReport( + int OverallScore, + string Grade, + IReadOnlyList Components, + IReadOnlyList Recommendations, + DateTimeOffset AssessedAt); + +public static class TelemetryQualityAssessor +{ + public static TelemetryQualityReport Assess( + QualityAssessmentInput input, + DateTimeOffset? now = null) + { + ArgumentNullException.ThrowIfNull(input); + DateTimeOffset assessedAt = now ?? DateTimeOffset.UtcNow; + TimeSpan freshnessWindow = input.FreshnessWindow ?? TimeSpan.FromHours(24); + + QualityComponentScore coverage = ScoreCoverage(input.Sources, assessedAt, freshnessWindow); + QualityComponentScore completeness = ScoreCompleteness(input.Fields); + QualityComponentScore freshness = ScoreFreshness(input.Sources, assessedAt, freshnessWindow); + QualityComponentScore schema = ScoreSchema(input.SampleEvents ?? []); + + QualityComponentScore[] components = [coverage, completeness, freshness, schema]; + int totalWeight = components.Sum(item => item.Weight); + int overall = totalWeight == 0 + ? 0 + : (int)Math.Round( + components.Sum(item => item.Score * item.Weight) / (double)totalWeight, + MidpointRounding.AwayFromZero); + + List recommendations = []; + foreach (QualityComponentScore component in components.Where(item => item.Score < 70)) + { + recommendations.AddRange(component.Evidence.Select(item => $"{component.Name}: {item}")); + } + + if (recommendations.Count == 0) + { + recommendations.Add("Telemetry quality is within target thresholds."); + } + + return new TelemetryQualityReport( + overall, + Grade(overall), + components, + recommendations, + assessedAt); + } + + private static QualityComponentScore ScoreCoverage( + IReadOnlyList sources, + DateTimeOffset now, + TimeSpan freshnessWindow) + { + if (sources.Count == 0) + { + return new QualityComponentScore( + "coverage", + 0, + 30, + "No sources configured.", + ["Register required security sources."]); + } + + SourceCoverageObservation[] required = sources.Where(item => item.Required).ToArray(); + IReadOnlyList basis = required.Length > 0 ? required : sources; + int covered = basis.Count(item => + item.Healthy && + item.EventsObserved > 0 && + item.LastEventAt is { } last && + now - last <= freshnessWindow); + int score = (int)Math.Round(100.0 * covered / basis.Count, MidpointRounding.AwayFromZero); + List evidence = basis + .Where(item => !(item.Healthy && item.EventsObserved > 0)) + .Select(item => $"Source '{item.SourceId}' ({item.SourceType}) not contributing.") + .Take(5) + .ToList(); + + return new QualityComponentScore( + "coverage", + score, + 30, + $"{covered}/{basis.Count} required sources healthy with recent events.", + evidence); + } + + private static QualityComponentScore ScoreCompleteness( + IReadOnlyList fields) + { + FieldCompletenessObservation[] required = fields + .Where(item => item.Required && item.TotalCount > 0) + .ToArray(); + if (required.Length == 0) + { + return new QualityComponentScore( + "completeness", + 50, + 25, + "No required field observations supplied.", + ["Provide field completeness samples for actor, action, target and correlation."]); + } + + double average = required.Average(item => 100.0 * item.PresentCount / item.TotalCount); + int score = (int)Math.Round(average, MidpointRounding.AwayFromZero); + List evidence = required + .Where(item => item.PresentCount * 100.0 / item.TotalCount < 80) + .Select(item => + $"Field '{item.FieldName}' present in {item.PresentCount}/{item.TotalCount} events.") + .Take(5) + .ToList(); + + return new QualityComponentScore( + "completeness", + score, + 25, + "Average required-field fill rate.", + evidence); + } + + private static QualityComponentScore ScoreFreshness( + IReadOnlyList sources, + DateTimeOffset now, + TimeSpan freshnessWindow) + { + SourceCoverageObservation[] withEvents = sources + .Where(item => item.LastEventAt is not null) + .ToArray(); + if (withEvents.Length == 0) + { + return new QualityComponentScore( + "freshness", + 0, + 25, + "No source timestamps available.", + ["Confirm collectors are emitting heartbeats and events."]); + } + + int fresh = withEvents.Count(item => now - item.LastEventAt! <= freshnessWindow); + int score = (int)Math.Round(100.0 * fresh / withEvents.Length, MidpointRounding.AwayFromZero); + List evidence = withEvents + .Where(item => now - item.LastEventAt! > freshnessWindow) + .Select(item => $"Source '{item.SourceId}' last event at {item.LastEventAt:O}.") + .Take(5) + .ToList(); + + return new QualityComponentScore( + "freshness", + score, + 25, + $"{fresh}/{withEvents.Length} sources within freshness window {freshnessWindow}.", + evidence); + } + + private static QualityComponentScore ScoreSchema(IReadOnlyList events) + { + if (events.Count == 0) + { + return new QualityComponentScore( + "schema", + 50, + 20, + "No sample events supplied for schema quality.", + ["Include sample events to score actor/target/correlation completeness."]); + } + + int points = 0; + int total = events.Count * 4; + foreach (SecurityEventEnvelope envelope in events) + { + if (!string.IsNullOrWhiteSpace(envelope.Actor?.Username) || + !string.IsNullOrWhiteSpace(envelope.Actor?.UserId)) + { + points++; + } + + if (envelope.Target is not null) + { + points++; + } + + if (envelope.Request?.CorrelationId is not null || envelope.Request?.TraceId is not null) + { + points++; + } + + if (!string.IsNullOrWhiteSpace(envelope.EventAction)) + { + points++; + } + } + + int score = (int)Math.Round(100.0 * points / total, MidpointRounding.AwayFromZero); + List evidence = []; + if (score < 80) + { + evidence.Add("Increase actor, target, correlation and action population on emitted events."); + } + + return new QualityComponentScore( + "schema", + score, + 20, + "Semantic field population across sample events.", + evidence); + } + + private static string Grade(int score) + { + return score switch + { + >= 90 => "A", + >= 80 => "B", + >= 70 => "C", + >= 60 => "D", + _ => "F" + }; + } +} diff --git a/src/Bower.Analytics/packages.lock.json b/src/Bower.Analytics/packages.lock.json new file mode 100644 index 0000000..4e48e5e --- /dev/null +++ b/src/Bower.Analytics/packages.lock.json @@ -0,0 +1,10 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "bower.contracts": { + "type": "Project" + } + } + } +} \ No newline at end of file diff --git a/tests/Bower.UnitTests/Bower.UnitTests.csproj b/tests/Bower.UnitTests/Bower.UnitTests.csproj index 02e4547..0ed4a57 100644 --- a/tests/Bower.UnitTests/Bower.UnitTests.csproj +++ b/tests/Bower.UnitTests/Bower.UnitTests.csproj @@ -36,5 +36,6 @@ + diff --git a/tests/Bower.UnitTests/TelemetryQualityAssessorTests.cs b/tests/Bower.UnitTests/TelemetryQualityAssessorTests.cs new file mode 100644 index 0000000..c18b620 --- /dev/null +++ b/tests/Bower.UnitTests/TelemetryQualityAssessorTests.cs @@ -0,0 +1,55 @@ +using Bower.Analytics; +using Bower.Contracts; + +namespace Bower.UnitTests; + +public sealed class TelemetryQualityAssessorTests +{ + [Fact] + public void Assess_HealthyFleet_ScoresHigh() + { + DateTimeOffset now = DateTimeOffset.Parse("2026-07-28T00:00:00Z", System.Globalization.CultureInfo.InvariantCulture); + QualityAssessmentInput input = new( + [ + new SourceCoverageObservation("http", "http-collector", true, now.AddMinutes(-5), 100, true), + new SourceCoverageObservation("sql", "sqlserver", true, now.AddMinutes(-10), 50, true) + ], + [ + new FieldCompletenessObservation("actor.username", 95, 100, true), + new FieldCompletenessObservation("target.name", 90, 100, true), + new FieldCompletenessObservation("request.correlationId", 88, 100, true) + ], + [ + new SecurityEventEnvelope + { + SchemaVersion = SecurityEventEnvelope.CurrentSchemaVersion, + EventId = Guid.CreateVersion7().ToString(), + TimeGenerated = now, + EventCategory = SecurityEventCategories.Authentication, + EventType = SecurityEventTypes.AuthenticationFailure, + EventAction = "authentication.attempt", + EventResult = EventResult.Failure, + Application = new ApplicationContext { Name = "app", Environment = "test" }, + Actor = new ActorContext { Username = "alice" }, + Target = new TargetContext { Type = "account", Name = "alice" }, + Request = new RequestContext { CorrelationId = "c1" } + } + ]); + + TelemetryQualityReport report = TelemetryQualityAssessor.Assess(input, now); + + Assert.True(report.OverallScore >= 85); + Assert.True(report.Grade is "A" or "B"); + Assert.Equal(4, report.Components.Count); + } + + [Fact] + public void Assess_MissingSources_RecommendsCoverage() + { + TelemetryQualityReport report = TelemetryQualityAssessor.Assess( + new QualityAssessmentInput([], [])); + + Assert.Equal(0, report.Components.Single(item => item.Name == "coverage").Score); + Assert.Contains(report.Recommendations, item => item.Contains("sources", StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/tests/Bower.UnitTests/packages.lock.json b/tests/Bower.UnitTests/packages.lock.json index f7aca87..4af3f1e 100644 --- a/tests/Bower.UnitTests/packages.lock.json +++ b/tests/Bower.UnitTests/packages.lock.json @@ -414,6 +414,12 @@ "Bower.Contracts": "[1.0.0, )" } }, + "bower.analytics": { + "type": "Project", + "dependencies": { + "Bower.Contracts": "[1.0.0, )" + } + }, "bower.contracts": { "type": "Project" },