update openapi de-serialize

Project.toml

@@ -1,14 +1,15 @@
 name = "InfrastructureSystems"
 uuid = "2cd47ed4-ca9b-11e9-27f2-ab636a7671f1"
-authors = ["Daniel Thom", "Jose Daniel Lara", "Gabriel Konar-Steenberg", "Clayton Barrows"]
 version = "3.6.0"
+authors = ["Daniel Thom", "Jose Daniel Lara", "Gabriel Konar-Steenberg", "Clayton Barrows"]
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+GeoJSON = "61d90e0f-e114-555e-ac52-39dfb47a3ef9"
 H5Zblosc = "c8ec2601-a99c-407f-b158-e79c03c2f5f7"
 HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
@@ -17,6 +18,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 Mustache = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+PowerCoreOpenAPIModels = "b7b40286-e793-417d-a9a0-b1583e4da1cb"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
@@ -28,16 +30,24 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed"
 TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e"
+TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
 
+[sources]
+# Git rev, not a sibling path: a relative path resolves only in a workspace checkout and
+# fails outright in CI, which clones this repo alone. Pinned to a branch rather than a
+# version because PowerOpenAPIModels is not registered.
+PowerCoreOpenAPIModels = {url = "https://github.com/Sienna-Platform/PowerOpenAPIModels.git", rev = "main", subdir = "PowerCoreOpenAPIModels.jl"}
+
 [compat]
 CSV = "0.9, 0.10"
 DataFrames = "^1.7"
 DataStructures = "^0.18, ^0.19"
 Dates = "1"
 DocStringExtensions = "0.8, 0.9"
+GeoJSON = "0.8.4"
 H5Zblosc = "0.1"
 HDF5 = "0.17"
 InteractiveUtils = "1"
@@ -46,6 +56,7 @@ LinearAlgebra = "1"
 Logging = "1"
 Mustache = "1"
 Pkg = "1"
+PowerCoreOpenAPIModels = "0.1"
 PrettyTables = "3.1"
 Printf = "1"
 Random = "1"
@@ -57,6 +68,7 @@ TOML = "1"
 Tables = "^1.11"
 TerminalLoggers = "~0.1"
 TimeSeries = "^0.25, ^0.26"
+TimeZones = "1"
 TimerOutputs = "^0.5"
 UUIDs = "1"
 YAML = "~0.4"

src/InfrastructureSystems.jl

@@ -23,6 +23,9 @@ import DataFrames
 import DataFrames: DataFrame
 import Dates
 import JSON
+import TimeZones
+import GeoJSON
+import PowerCoreOpenAPIModels
 import Logging
 import Random
 import Pkg
@@ -186,6 +189,7 @@ include("component_container.jl")
 include("component_uuids.jl")
 include("geographic_supplemental_attribute.jl")
 include("data_source_supplemental_attribute.jl")
+include("openapi_converters.jl")
 include("generated/includes.jl")
 include("time_series_parser.jl")
 include("single_time_series.jl")

src/geographic_supplemental_attribute.jl

@@ -12,6 +12,66 @@ Supplemental attribute to store geographic information about system components i
 struct GeographicInfo <: SupplementalAttribute
     geo_json::Dict{String, Any}
     internal::InfrastructureSystemsInternal
+
+    function GeographicInfo(geo_json::Dict{String, <:Any}, internal)
+        validate_geo_json(geo_json)
+        return new(geo_json, internal)
+    end
+end
+
+"""
+Whether a parsed GeoJSON object carries the member its type requires.
+
+Parsing alone does not establish this: `GeoJSON.read` accepts `{"type": "Point"}` with no
+`coordinates` at all, and `{"type": "Feature"}` with no `geometry`, returning an object whose
+required member is `nothing` rather than raising. Both are structurally invalid GeoJSON, so
+they are rejected here instead of reaching a consumer that then has to cope with a geometry
+that has no coordinates.
+
+Dispatched per shape rather than branching on `type`, so each accessor is only ever applied
+to the object it is defined for. The fallback is `true` — parsing succeeded and the shape is
+one this function does not have a rule for, which is not grounds to reject it.
+"""
+_geo_json_complete(x::GeoJSON.AbstractGeometry) = !isnothing(GeoJSON.coordinates(x))
+_geo_json_complete(x::GeoJSON.GeometryCollection) = !isnothing(GeoJSON.geometry(x))
+_geo_json_complete(x::GeoJSON.Feature) = !isnothing(GeoJSON.geometry(x))
+_geo_json_complete(x::GeoJSON.AbstractFeatureCollection) = !isnothing(GeoJSON.features(x))
+_geo_json_complete(::Any) = true
+
+"""
+Check that `geo_json` is actually GeoJSON, by parsing it and confirming it is structurally
+complete.
+
+An empty dictionary passes: it is the documented default and means no geographic information
+was recorded, which is different from recording something malformed. Anything else must parse
+as a GeoJSON object *and* carry the member its type requires, so a typo'd `type`, a coordinate
+list of the wrong shape, and a geometry missing its coordinates outright are all caught where
+they are introduced rather than surfacing much later in whatever consumes the geometry.
+
+The parse result is discarded — `GeoJSON` reads coordinates as `Float32`, and the stored
+value stays the caller's own dictionary rather than a lossy round-trip of it.
+"""
+function validate_geo_json(geo_json::Dict{String, <:Any})
+    isempty(geo_json) && return nothing
+    parsed = try
+        GeoJSON.read(JSON.json(geo_json))
+    catch e
+        throw(
+            ArgumentError(
+                "GeographicInfo.geo_json is not valid GeoJSON: $(sprint(showerror, e)). " *
+                "Pass an empty dictionary to record no geographic information.",
+            ),
+        )
+    end
+    _geo_json_complete(parsed) || throw(
+        ArgumentError(
+            "GeographicInfo.geo_json parses as $(nameof(typeof(parsed))) but is missing the " *
+            "member that type requires (`coordinates` for a geometry, `geometry` for a " *
+            "Feature, `features` for a FeatureCollection). Pass an empty dictionary to " *
+            "record no geographic information.",
+        ),
+    )
+    return nothing
 end
 
 """

src/openapi_converters.jl

@@ -0,0 +1,101 @@
+# OpenAPI serde for the supplemental attributes InfrastructureSystems itself owns.
+#
+# `GeographicInfo` and `DataSource` are IS types, so their field mapping belongs here rather
+# than in a domain package: PowerSystems previously carried the `GeographicInfo` pair only
+# because it happened to own the document walk, which meant a second domain package wanting
+# the same attribute would have had to duplicate it.
+#
+# These take no `OpenAPIRefs`. That registry lives in the domain package because it carries
+# the document's `unit_system` and `base_power` — power-domain state IS has no notion of —
+# so the export direction takes the id its caller already resolved. Neither type has a
+# unit-bearing field, so there is nothing else the domain layer would need to supply.
+
+"""
+Convert an OpenAPI-model instance into the matching Sienna type.
+
+Declared here, extended by domain packages: `PowerSystems` adds a method per component and
+per attribute it owns, so `from_openapi` stays one function across the stack rather than one
+per package.
+"""
+function from_openapi end
+
+"""
+Convert a Sienna component or attribute into its OpenAPI-model representation.
+
+The counterpart of [`from_openapi`](@ref); the same extension rule applies.
+"""
+function to_openapi end
+
+# ── GeographicInfo ──────────────────────────────────────────────────────────────
+
+from_openapi(po::PowerCoreOpenAPIModels.GeographicInfo) =
+    GeographicInfo(; geo_json = po.geo_json)
+
+to_openapi(geo::GeographicInfo, id::Int) =
+    PowerCoreOpenAPIModels.GeographicInfo(; id = id, geo_json = get_geo_json(geo))
+
+# ── DataSource ──────────────────────────────────────────────────────────────────
+#
+# The document states both timestamps as `ZonedDateTime` while `DataSource` stores plain
+# `DateTime`, so import drops the offset after normalizing to UTC and export re-attaches
+# UTC. Normalizing rather than discarding the zone matters: two documents recording the same
+# instant in different zones must import to the same `DateTime`, which `DateTime(zdt)` alone
+# would not give.
+#
+# `extra` widens `Dict{String, String}` to the `Dict{String, Any}` the field declares; on the
+# way out, values are stringified, since the schema types that map as strings.
+
+_datasource_utc(zdt) = Dates.DateTime(TimeZones.astimezone(zdt, TimeZones.tz"UTC"))
+_datasource_utc(::Nothing) = nothing
+
+_datasource_zoned(dt::Dates.DateTime) = TimeZones.ZonedDateTime(dt, TimeZones.tz"UTC")
+
+# `published_at`/`recorded_by` are absence-by-predicate, not absence-by-`nothing`: their
+# accessors error rather than return a sentinel, so the export path asks first. An absent
+# field is written as `null`, which the schema marks optional.
+function _datasource_published_at(ds::DataSource)
+    has_published_at(ds) || return nothing
+    return _datasource_zoned(get_published_at(ds))
+end
+
+function _datasource_recorded_by(ds::DataSource)
+    has_recorded_by(ds) || return nothing
+    return get_recorded_by(ds)
+end
+
+_datasource_fields(::Nothing) = String[]
+_datasource_fields(v) = collect(String, v)
+
+_datasource_extra(::Nothing) = Dict{String, Any}()
+_datasource_extra(d) = Dict{String, Any}(k => v for (k, v) in d)
+
+function from_openapi(po::PowerCoreOpenAPIModels.DataSource)
+    return DataSource(;
+        organization = po.organization,
+        retrieved_at = _datasource_utc(po.retrieved_at),
+        dataset = po.dataset,
+        url = po.url,
+        version = po.version,
+        published_at = _datasource_utc(po.published_at),
+        confidence = po.confidence,
+        recorded_by = po.recorded_by,
+        fields = _datasource_fields(po.fields),
+        extra = _datasource_extra(po.extra),
+    )
+end
+
+function to_openapi(ds::DataSource, id::Int)
+    return PowerCoreOpenAPIModels.DataSource(;
+        id = id,
+        organization = get_organization(ds),
+        retrieved_at = _datasource_zoned(get_retrieved_at(ds)),
+        dataset = get_dataset(ds),
+        url = get_url(ds),
+        version = get_version(ds),
+        published_at = _datasource_published_at(ds),
+        confidence = get_confidence(ds),
+        recorded_by = _datasource_recorded_by(ds),
+        fields = get_fields(ds),
+        extra = Dict{String, String}(k => string(v) for (k, v) in get_extra(ds)),
+    )
+end

test/test_serialization.jl

@@ -153,7 +153,8 @@ end
     resolution = Dates.Hour(1)
     ta = TimeSeries.TimeArray(range(initial_time; length = 24, step = resolution), rand(24))
     ts = IS.SingleTimeSeries(; data = ta, name = "test")
-    geo = IS.GeographicInfo(; geo_json = Dict("x" => 1.0, "y" => 2.0))
+    geo =
+        IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [1.0, 2.0]))
 
     for i in 1:2
         name = "component_$(i)"

test/test_supplemental_attributes.jl

@@ -15,8 +15,10 @@ end
 
 @testset "Test bulk addition of supplemental attributes" begin
     mgr = IS.SupplementalAttributeManager()
-    attr1 = IS.GeographicInfo(; geo_json = Dict("x" => 1.0))
-    attr2 = IS.GeographicInfo(; geo_json = Dict("x" => 2.0))
+    attr1 =
+        IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [1.0, 0.0]))
+    attr2 =
+        IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [2.0, 0.0]))
     component = IS.TestComponent("component1", 1)
     IS.begin_supplemental_attributes_update(mgr) do
         IS.add_supplemental_attribute!(mgr, component, attr1)
@@ -30,7 +32,8 @@ end
 @testset "Test bulk addition of supplemental attributes with error" begin
     mgr = IS.SupplementalAttributeManager()
     attr1 = IS.TestSupplemental(; value = 1.0)
-    attr2 = IS.GeographicInfo(; geo_json = Dict("x" => 2.0))
+    attr2 =
+        IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [2.0, 0.0]))
     component = IS.TestComponent("component1", 1)
     @test_throws(
         ArgumentError,
@@ -47,15 +50,17 @@ end
 @testset "Test bulk addition of supplemental attributes with error, existing attrs" begin
     mgr = IS.SupplementalAttributeManager()
     attr1 = IS.TestSupplemental(; value = 1.0)
-    attr2 = IS.GeographicInfo(; geo_json = Dict("x" => 2.0))
+    attr2 =
+        IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [2.0, 0.0]))
     component = IS.TestComponent("component1", 1)
     IS.begin_supplemental_attributes_update(mgr) do
         IS.add_supplemental_attribute!(mgr, component, attr1)
         IS.add_supplemental_attribute!(mgr, component, attr2)
     end
 
     attr3 = IS.TestSupplemental(; value = 3.0)
-    attr4 = IS.GeographicInfo(; geo_json = Dict("x" => 3.0))
+    attr4 =
+        IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [3.0, 0.0]))
     @test_throws(
         ArgumentError,
         IS.begin_supplemental_attributes_update(mgr) do
@@ -74,7 +79,8 @@ end
     mgr = IS.SupplementalAttributeManager()
     attr1 = IS.TestSupplemental(; value = 1.0)
     attr2 = IS.TestSupplemental(; value = 2.0)
-    attr3 = IS.GeographicInfo(; geo_json = Dict("x" => 3.0))
+    attr3 =
+        IS.GeographicInfo(; geo_json = Dict("type" => "Point", "coordinates" => [3.0, 0.0]))
     component = IS.TestComponent("component1", 1)
     IS.begin_supplemental_attributes_update(mgr) do
         IS.add_supplemental_attribute!(mgr, component, attr1)
@@ -342,24 +348,28 @@ end
 
 @testset "supplemental attribute transaction rolls back in-place mutations" begin
     mgr = IS.SupplementalAttributeManager()
-    attr = IS.GeographicInfo(; geo_json = Dict{String, Any}("x" => 1.0))
+    attr = IS.GeographicInfo(;
+        geo_json = Dict{String, Any}("type" => "Point", "coordinates" => [1.0, 0.0]),
+    )
     component = IS.TestComponent("component1", 1)
     IS.add_supplemental_attribute!(mgr, component, attr)
 
     @test_throws ErrorException IS.begin_supplemental_attributes_update(mgr) do
         stored = IS.get_supplemental_attribute(mgr, IS.get_uuid(attr))
-        IS.get_geo_json(stored)["x"] = 999.0
+        IS.get_geo_json(stored)["coordinates"] = [999.0, 0.0]
         error("boom")
     end
 
     stored = IS.get_supplemental_attribute(mgr, IS.get_uuid(attr))
-    @test IS.get_geo_json(stored)["x"] == 1.0
+    @test IS.get_geo_json(stored)["coordinates"] == [1.0, 0.0]
 end
 
 @testset "supplemental attribute rollback does not clone the manager" begin
     data = IS.SystemData()
     mgr = data.supplemental_attribute_manager
-    attr = IS.GeographicInfo(; geo_json = Dict{String, Any}("x" => 1.0))
+    attr = IS.GeographicInfo(;
+        geo_json = Dict{String, Any}("type" => "Point", "coordinates" => [1.0, 0.0]),
+    )
     component = IS.TestComponent("component1", 1)
     IS.add_component!(data, component)
     IS.add_supplemental_attribute!(data, component, attr)

test/test_system_data.jl

@@ -400,7 +400,9 @@ end
     attr1 = IS.TestSupplemental(; value = 1.0)
     attr2 = IS.TestSupplemental(; value = 2.0)
     geo_attr1 = IS.GeographicInfo()
-    geo_attr2 = IS.GeographicInfo(; geo_json = Dict{String, Any}("foo" => 5))
+    geo_attr2 = IS.GeographicInfo(;
+        geo_json = Dict{String, Any}("type" => "Point", "coordinates" => [5.0, 0.0]),
+    )
     components_to_attributes = Dict(
         (IS.TestComponent, "component1") => [geo_attr1, attr1],
         (IS.TestComponent, "component2") => [attr1],