diff --git a/Project.toml b/Project.toml index e453694a2..6b0e7e6d3 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "InfrastructureSystems" uuid = "2cd47ed4-ca9b-11e9-27f2-ab636a7671f1" -authors = ["Daniel Thom", "Jose Daniel Lara", "Gabriel Konar-Steenberg", "Clayton Barrows"] version = "3.6.0" +authors = ["Daniel Thom", "Jose Daniel Lara", "Gabriel Konar-Steenberg", "Clayton Barrows"] [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" @@ -9,6 +9,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +GeoJSON = "61d90e0f-e114-555e-ac52-39dfb47a3ef9" H5Zblosc = "c8ec2601-a99c-407f-b158-e79c03c2f5f7" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" @@ -17,6 +18,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" Mustache = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +PowerCoreOpenAPIModels = "b7b40286-e793-417d-a9a0-b1583e4da1cb" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -28,16 +30,24 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed" TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e" +TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" +[sources] +# Git rev, not a sibling path: a relative path resolves only in a workspace checkout and +# fails outright in CI, which clones this repo alone. Pinned to a branch rather than a +# version because PowerOpenAPIModels is not registered. +PowerCoreOpenAPIModels = {url = "https://github.com/Sienna-Platform/PowerOpenAPIModels.git", rev = "main", subdir = "PowerCoreOpenAPIModels.jl"} + [compat] CSV = "0.9, 0.10" DataFrames = "^1.7" DataStructures = "^0.18, ^0.19" Dates = "1" DocStringExtensions = "0.8, 0.9" +GeoJSON = "0.8.4" H5Zblosc = "0.1" HDF5 = "0.17" InteractiveUtils = "1" @@ -46,6 +56,7 @@ LinearAlgebra = "1" Logging = "1" Mustache = "1" Pkg = "1" +PowerCoreOpenAPIModels = "0.1" PrettyTables = "3.1" Printf = "1" Random = "1" @@ -57,6 +68,7 @@ TOML = "1" Tables = "^1.11" TerminalLoggers = "~0.1" TimeSeries = "^0.25, ^0.26" +TimeZones = "1" TimerOutputs = "^0.5" UUIDs = "1" YAML = "~0.4" diff --git a/src/InfrastructureSystems.jl b/src/InfrastructureSystems.jl index a86cc114a..36a275431 100644 --- a/src/InfrastructureSystems.jl +++ b/src/InfrastructureSystems.jl @@ -23,6 +23,9 @@ import DataFrames import DataFrames: DataFrame import Dates import JSON +import TimeZones +import GeoJSON +import PowerCoreOpenAPIModels import Logging import Random import Pkg @@ -186,6 +189,7 @@ include("component_container.jl") include("component_uuids.jl") include("geographic_supplemental_attribute.jl") include("data_source_supplemental_attribute.jl") +include("openapi_converters.jl") include("generated/includes.jl") include("time_series_parser.jl") include("single_time_series.jl") diff --git a/src/geographic_supplemental_attribute.jl b/src/geographic_supplemental_attribute.jl index bcfe30c62..2ac8b202b 100644 --- a/src/geographic_supplemental_attribute.jl +++ b/src/geographic_supplemental_attribute.jl @@ -12,6 +12,66 @@ Supplemental attribute to store geographic information about system components i struct GeographicInfo <: SupplementalAttribute geo_json::Dict{String, Any} internal::InfrastructureSystemsInternal + + function GeographicInfo(geo_json::Dict{String, <:Any}, internal) + validate_geo_json(geo_json) + return new(geo_json, internal) + end +end + +""" +Whether a parsed GeoJSON object carries the member its type requires. + +Parsing alone does not establish this: `GeoJSON.read` accepts `{"type": "Point"}` with no +`coordinates` at all, and `{"type": "Feature"}` with no `geometry`, returning an object whose +required member is `nothing` rather than raising. Both are structurally invalid GeoJSON, so +they are rejected here instead of reaching a consumer that then has to cope with a geometry +that has no coordinates. + +Dispatched per shape rather than branching on `type`, so each accessor is only ever applied +to the object it is defined for. The fallback is `true` — parsing succeeded and the shape is +one this function does not have a rule for, which is not grounds to reject it. +""" +_geo_json_complete(x::GeoJSON.AbstractGeometry) = !isnothing(GeoJSON.coordinates(x)) +_geo_json_complete(x::GeoJSON.GeometryCollection) = !isnothing(GeoJSON.geometry(x)) +_geo_json_complete(x::GeoJSON.Feature) = !isnothing(GeoJSON.geometry(x)) +_geo_json_complete(x::GeoJSON.AbstractFeatureCollection) = !isnothing(GeoJSON.features(x)) +_geo_json_complete(::Any) = true + +""" +Check that `geo_json` is actually GeoJSON, by parsing it and confirming it is structurally +complete. + +An empty dictionary passes: it is the documented default and means no geographic information +was recorded, which is different from recording something malformed. Anything else must parse +as a GeoJSON object *and* carry the member its type requires, so a typo'd `type`, a coordinate +list of the wrong shape, and a geometry missing its coordinates outright are all caught where +they are introduced rather than surfacing much later in whatever consumes the geometry. + +The parse result is discarded — `GeoJSON` reads coordinates as `Float32`, and the stored +value stays the caller's own dictionary rather than a lossy round-trip of it. +""" +function validate_geo_json(geo_json::Dict{String, <:Any}) + isempty(geo_json) && return nothing + parsed = try + GeoJSON.read(JSON.json(geo_json)) + catch e + throw( + ArgumentError( + "GeographicInfo.geo_json is not valid GeoJSON: $(sprint(showerror, e)). " * + "Pass an empty dictionary to record no geographic information.", + ), + ) + end + _geo_json_complete(parsed) || throw( + ArgumentError( + "GeographicInfo.geo_json parses as $(nameof(typeof(parsed))) but is missing the " * + "member that type requires (`coordinates` for a geometry, `geometry` for a " * + "Feature, `features` for a FeatureCollection). Pass an empty dictionary to " * + "record no geographic information.", + ), + ) + return nothing end """ diff --git a/src/openapi_converters.jl b/src/openapi_converters.jl new file mode 100644 index 000000000..fc2feb617 --- /dev/null +++ b/src/openapi_converters.jl @@ -0,0 +1,101 @@ +# OpenAPI serde for the supplemental attributes InfrastructureSystems itself owns. +# +# `GeographicInfo` and `DataSource` are IS types, so their field mapping belongs here rather +# than in a domain package: PowerSystems previously carried the `GeographicInfo` pair only +# because it happened to own the document walk, which meant a second domain package wanting +# the same attribute would have had to duplicate it. +# +# These take no `OpenAPIRefs`. That registry lives in the domain package because it carries +# the document's `unit_system` and `base_power` — power-domain state IS has no notion of — +# so the export direction takes the id its caller already resolved. Neither type has a +# unit-bearing field, so there is nothing else the domain layer would need to supply. + +""" +Convert an OpenAPI-model instance into the matching Sienna type. + +Declared here, extended by domain packages: `PowerSystems` adds a method per component and +per attribute it owns, so `from_openapi` stays one function across the stack rather than one +per package. +""" +function from_openapi end + +""" +Convert a Sienna component or attribute into its OpenAPI-model representation. + +The counterpart of [`from_openapi`](@ref); the same extension rule applies. +""" +function to_openapi end + +# ── GeographicInfo ────────────────────────────────────────────────────────────── + +from_openapi(po::PowerCoreOpenAPIModels.GeographicInfo) = + GeographicInfo(; geo_json = po.geo_json) + +to_openapi(geo::GeographicInfo, id::Int) = + PowerCoreOpenAPIModels.GeographicInfo(; id = id, geo_json = get_geo_json(geo)) + +# ── DataSource ────────────────────────────────────────────────────────────────── +# +# The document states both timestamps as `ZonedDateTime` while `DataSource` stores plain +# `DateTime`, so import drops the offset after normalizing to UTC and export re-attaches +# UTC. Normalizing rather than discarding the zone matters: two documents recording the same +# instant in different zones must import to the same `DateTime`, which `DateTime(zdt)` alone +# would not give. +# +# `extra` widens `Dict{String, String}` to the `Dict{String, Any}` the field declares; on the +# way out, values are stringified, since the schema types that map as strings. + +_datasource_utc(zdt) = Dates.DateTime(TimeZones.astimezone(zdt, TimeZones.tz"UTC")) +_datasource_utc(::Nothing) = nothing + +_datasource_zoned(dt::Dates.DateTime) = TimeZones.ZonedDateTime(dt, TimeZones.tz"UTC") + +# `published_at`/`recorded_by` are absence-by-predicate, not absence-by-`nothing`: their +# accessors error rather than return a sentinel, so the export path asks first. An absent +# field is written as `null`, which the schema marks optional. +function _datasource_published_at(ds::DataSource) + has_published_at(ds) || return nothing + return _datasource_zoned(get_published_at(ds)) +end + +function _datasource_recorded_by(ds::DataSource) + has_recorded_by(ds) || return nothing + return get_recorded_by(ds) +end + +_datasource_fields(::Nothing) = String[] +_datasource_fields(v) = collect(String, v) + +_datasource_extra(::Nothing) = Dict{String, Any}() +_datasource_extra(d) = Dict{String, Any}(k => v for (k, v) in d) + +function from_openapi(po::PowerCoreOpenAPIModels.DataSource) + return DataSource(; + organization = po.organization, + retrieved_at = _datasource_utc(po.retrieved_at), + dataset = po.dataset, + url = po.url, + version = po.version, + published_at = _datasource_utc(po.published_at), + confidence = po.confidence, + recorded_by = po.recorded_by, + fields = _datasource_fields(po.fields), + extra = _datasource_extra(po.extra), + ) +end + +function to_openapi(ds::DataSource, id::Int) + return PowerCoreOpenAPIModels.DataSource(; + id = id, + organization = get_organization(ds), + retrieved_at = _datasource_zoned(get_retrieved_at(ds)), + dataset = get_dataset(ds), + url = get_url(ds), + version = get_version(ds), + published_at = _datasource_published_at(ds), + confidence = get_confidence(ds), + recorded_by = _datasource_recorded_by(ds), + fields = get_fields(ds), + extra = Dict{String, String}(k => string(v) for (k, v) in get_extra(ds)), + ) +end diff --git a/test/test_serialization.jl b/test/test_serialization.jl index 77e154633..35db0f35e 100644 --- a/test/test_serialization.jl +++ b/test/test_serialization.jl @@ -153,7 +153,8 @@ end resolution = Dates.Hour(1) ta = TimeSeries.TimeArray(range(initial_time; length = 24, step = resolution), rand(24)) ts = IS.SingleTimeSeries(; data = ta, name = "test") - geo = IS.GeographicInfo(; geo_json = Dict("x" => 1.0, "y" => 2.0)) + geo = + IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [1.0, 2.0])) for i in 1:2 name = "component_$(i)" diff --git a/test/test_supplemental_attributes.jl b/test/test_supplemental_attributes.jl index a7d25908f..94516b46a 100644 --- a/test/test_supplemental_attributes.jl +++ b/test/test_supplemental_attributes.jl @@ -15,8 +15,10 @@ end @testset "Test bulk addition of supplemental attributes" begin mgr = IS.SupplementalAttributeManager() - attr1 = IS.GeographicInfo(; geo_json = Dict("x" => 1.0)) - attr2 = IS.GeographicInfo(; geo_json = Dict("x" => 2.0)) + attr1 = + IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [1.0, 0.0])) + attr2 = + IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [2.0, 0.0])) component = IS.TestComponent("component1", 1) IS.begin_supplemental_attributes_update(mgr) do IS.add_supplemental_attribute!(mgr, component, attr1) @@ -30,7 +32,8 @@ end @testset "Test bulk addition of supplemental attributes with error" begin mgr = IS.SupplementalAttributeManager() attr1 = IS.TestSupplemental(; value = 1.0) - attr2 = IS.GeographicInfo(; geo_json = Dict("x" => 2.0)) + attr2 = + IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [2.0, 0.0])) component = IS.TestComponent("component1", 1) @test_throws( ArgumentError, @@ -47,7 +50,8 @@ end @testset "Test bulk addition of supplemental attributes with error, existing attrs" begin mgr = IS.SupplementalAttributeManager() attr1 = IS.TestSupplemental(; value = 1.0) - attr2 = IS.GeographicInfo(; geo_json = Dict("x" => 2.0)) + attr2 = + IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [2.0, 0.0])) component = IS.TestComponent("component1", 1) IS.begin_supplemental_attributes_update(mgr) do IS.add_supplemental_attribute!(mgr, component, attr1) @@ -55,7 +59,8 @@ end end attr3 = IS.TestSupplemental(; value = 3.0) - attr4 = IS.GeographicInfo(; geo_json = Dict("x" => 3.0)) + attr4 = + IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [3.0, 0.0])) @test_throws( ArgumentError, IS.begin_supplemental_attributes_update(mgr) do @@ -74,7 +79,8 @@ end mgr = IS.SupplementalAttributeManager() attr1 = IS.TestSupplemental(; value = 1.0) attr2 = IS.TestSupplemental(; value = 2.0) - attr3 = IS.GeographicInfo(; geo_json = Dict("x" => 3.0)) + attr3 = + IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [3.0, 0.0])) component = IS.TestComponent("component1", 1) IS.begin_supplemental_attributes_update(mgr) do IS.add_supplemental_attribute!(mgr, component, attr1) @@ -342,24 +348,28 @@ end @testset "supplemental attribute transaction rolls back in-place mutations" begin mgr = IS.SupplementalAttributeManager() - attr = IS.GeographicInfo(; geo_json = Dict{String, Any}("x" => 1.0)) + attr = IS.GeographicInfo(; + geo_json = Dict{String, Any}("type" => "Point", "coordinates" => [1.0, 0.0]), + ) component = IS.TestComponent("component1", 1) IS.add_supplemental_attribute!(mgr, component, attr) @test_throws ErrorException IS.begin_supplemental_attributes_update(mgr) do stored = IS.get_supplemental_attribute(mgr, IS.get_uuid(attr)) - IS.get_geo_json(stored)["x"] = 999.0 + IS.get_geo_json(stored)["coordinates"] = [999.0, 0.0] error("boom") end stored = IS.get_supplemental_attribute(mgr, IS.get_uuid(attr)) - @test IS.get_geo_json(stored)["x"] == 1.0 + @test IS.get_geo_json(stored)["coordinates"] == [1.0, 0.0] end @testset "supplemental attribute rollback does not clone the manager" begin data = IS.SystemData() mgr = data.supplemental_attribute_manager - attr = IS.GeographicInfo(; geo_json = Dict{String, Any}("x" => 1.0)) + attr = IS.GeographicInfo(; + geo_json = Dict{String, Any}("type" => "Point", "coordinates" => [1.0, 0.0]), + ) component = IS.TestComponent("component1", 1) IS.add_component!(data, component) IS.add_supplemental_attribute!(data, component, attr) diff --git a/test/test_system_data.jl b/test/test_system_data.jl index b3c01a4cb..4fcb7f660 100644 --- a/test/test_system_data.jl +++ b/test/test_system_data.jl @@ -400,7 +400,9 @@ end attr1 = IS.TestSupplemental(; value = 1.0) attr2 = IS.TestSupplemental(; value = 2.0) geo_attr1 = IS.GeographicInfo() - geo_attr2 = IS.GeographicInfo(; geo_json = Dict{String, Any}("foo" => 5)) + geo_attr2 = IS.GeographicInfo(; + geo_json = Dict{String, Any}("type" => "Point", "coordinates" => [5.0, 0.0]), + ) components_to_attributes = Dict( (IS.TestComponent, "component1") => [geo_attr1, attr1], (IS.TestComponent, "component2") => [attr1],