Skip to content

Optimise schema framing when dealing with big schemas #2188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jviotti merged 1 commit into from
Jan 15, 2026
Merged

Optimise schema framing when dealing with big schemas #2188

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 106 additions & 66 deletions src/core/jsonschema/frame.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,11 +125,9 @@ auto find_anchors(const sourcemeta::core::JSON &schema,
return result;
}

template <typename StringType>
auto find_nearest_bases_ref(
const std::unordered_map<sourcemeta::core::WeakPointer,
std::vector<StringType>> &bases,
const sourcemeta::core::WeakPointer &pointer)
template <typename StringType, typename MapType>
auto find_nearest_bases_ref(const MapType &bases,
const sourcemeta::core::WeakPointer &pointer)
-> std::optional<
std::pair<std::reference_wrapper<const std::vector<StringType>>,
sourcemeta::core::WeakPointer>> {
Expand All @@ -150,14 +148,12 @@ auto find_nearest_bases_ref(
return std::nullopt;
}

template <typename StringType>
auto find_nearest_bases(
const std::unordered_map<sourcemeta::core::WeakPointer,
std::vector<StringType>> &bases,
const sourcemeta::core::WeakPointer &pointer,
const std::optional<std::string_view> &default_base)
template <typename StringType, typename MapType>
auto find_nearest_bases(const MapType &bases,
const sourcemeta::core::WeakPointer &pointer,
const std::optional<std::string_view> &default_base)
-> std::pair<std::vector<StringType>, sourcemeta::core::WeakPointer> {
const auto result{find_nearest_bases_ref(bases, pointer)};
const auto result{find_nearest_bases_ref<StringType>(bases, pointer)};
if (result.has_value()) {
return {result->first.get(), result->second};
}
Expand All @@ -170,21 +166,37 @@ auto find_nearest_bases(
return {{}, sourcemeta::core::empty_weak_pointer};
}

auto find_every_base(
const std::unordered_map<sourcemeta::core::WeakPointer,
std::vector<sourcemeta::core::JSON::String>>
&bases,
const sourcemeta::core::WeakPointer &pointer)
-> std::vector<std::pair<std::string_view, sourcemeta::core::WeakPointer>> {
template <typename DialectStringType> struct CombinedWalkResult {
std::optional<
std::pair<std::reference_wrapper<const std::vector<DialectStringType>>,
sourcemeta::core::WeakPointer>>
dialect_match;
std::vector<std::pair<std::string_view, sourcemeta::core::WeakPointer>>
result;
every_base;
};

template <typename DialectStringType, typename DialectMapType,
typename BaseMapType>
auto find_dialect_and_all_bases(const DialectMapType &base_dialects,
const BaseMapType &base_uris,
const sourcemeta::core::WeakPointer &pointer)
-> CombinedWalkResult<DialectStringType> {
CombinedWalkResult<DialectStringType> result;

auto current_pointer{pointer};
while (true) {
const auto match{bases.find(current_pointer)};
if (match != bases.cend()) {
for (const auto &base : match->second) {
result.emplace_back(std::string_view{base}, current_pointer);
if (!result.dialect_match.has_value()) {
const auto dialect_it{base_dialects.find(current_pointer)};
if (dialect_it != base_dialects.cend()) {
result.dialect_match =
std::make_pair(std::cref(dialect_it->second), current_pointer);
}
}

const auto base_it{base_uris.find(current_pointer)};
if (base_it != base_uris.cend()) {
for (const auto &base : base_it->second) {
result.every_base.emplace_back(std::string_view{base}, current_pointer);
}
}

Expand All @@ -195,10 +207,10 @@ auto find_every_base(
current_pointer = current_pointer.initial();
}

if (result.empty() ||
result.back().second != sourcemeta::core::empty_weak_pointer) {
result.emplace_back(std::string_view{},
sourcemeta::core::empty_weak_pointer);
if (result.every_base.empty() ||
result.every_base.back().second != sourcemeta::core::empty_weak_pointer) {
result.every_base.emplace_back(std::string_view{},
sourcemeta::core::empty_weak_pointer);
}

return result;
Expand Down Expand Up @@ -439,9 +451,9 @@ auto SchemaFrame::analyse(const JSON &root, const SchemaWalker &walker,
assert(std::unordered_set<WeakPointer>(paths.cbegin(), paths.cend()).size() ==
paths.size());
std::vector<InternalEntry> subschema_entries;
std::unordered_map<WeakPointer, CacheSubschema> subschemas;
std::unordered_map<WeakPointer, std::vector<JSON::String>> base_uris;
std::unordered_map<WeakPointer, std::vector<std::string_view>> base_dialects;
std::map<WeakPointer, CacheSubschema> subschemas;
std::map<WeakPointer, std::vector<JSON::String>> base_uris;
std::map<WeakPointer, std::vector<std::string_view>> base_dialects;

for (const auto &path : paths) {
// Passing paths that overlap is undefined behavior. No path should
Expand Down Expand Up @@ -547,7 +559,7 @@ auto SchemaFrame::analyse(const JSON &root, const SchemaWalker &walker,
// identifier, we ignore it as a traditional identifier and take
// care of it as an anchor
!is_pre_2019_09_location_independent_identifier) {
const auto bases{find_nearest_bases(
const auto bases{find_nearest_bases<JSON::String>(
base_uris, common_pointer_weak,
entry.id ? std::optional<std::string_view>{*entry.id}
: std::nullopt)};
Expand Down Expand Up @@ -613,7 +625,7 @@ auto SchemaFrame::analyse(const JSON &root, const SchemaWalker &walker,
sourcemeta::core::dialect(entry.common.subschema.get())};
if (!maybe_metaschema.empty()) {
sourcemeta::core::URI metaschema{maybe_metaschema};
const auto nearest_bases{find_nearest_bases(
const auto nearest_bases{find_nearest_bases<JSON::String>(
base_uris, common_pointer_weak,
entry.id ? std::optional<std::string_view>{*entry.id}
: std::nullopt)};
Expand All @@ -639,7 +651,7 @@ auto SchemaFrame::analyse(const JSON &root, const SchemaWalker &walker,
// Handle schema anchors
for (const auto &[name, type] : find_anchors(entry.common.subschema.get(),
entry.common.vocabularies)) {
const auto bases{find_nearest_bases(
const auto bases{find_nearest_bases<JSON::String>(
base_uris, common_pointer_weak,
entry.id ? std::optional<std::string_view>{*entry.id}
: std::nullopt)};
Expand Down Expand Up @@ -742,13 +754,41 @@ auto SchemaFrame::analyse(const JSON &root, const SchemaWalker &walker,
for (const auto &relative_pointer : pointers) {
const auto pointer_weak{path.concat(relative_pointer)};

const auto dialect_match{
find_nearest_bases_ref(base_dialects, pointer_weak)};
const auto &dialect_for_pointer{dialect_match.has_value()
? dialect_match->first.get().front()
: root_dialect};
const auto combined{find_dialect_and_all_bases<std::string_view>(
base_dialects, base_uris, pointer_weak)};
const auto &dialect_for_pointer{
combined.dialect_match.has_value()
? combined.dialect_match->first.get().front()
: root_dialect};
const auto &every_base_result{combined.every_base};

std::optional<std::pair<std::string_view, WeakPointer>> nearest_base_info;
for (const auto &entry : every_base_result) {
if (!entry.first.empty()) {
nearest_base_info = entry;
break;
}
}

auto every_base_result = find_every_base(base_uris, pointer_weak);
const auto subschema_it{subschemas.find(pointer_weak)};
const bool is_subschema{subschema_it != subschemas.cend()};
const auto nearest_base_depth =
nearest_base_info.has_value() ? nearest_base_info->second.size() : 0;

std::string_view hoisted_base_view{};
sourcemeta::core::SchemaBaseDialect hoisted_base_dialect{};
if (nearest_base_info.has_value()) {
const JSON::String nearest_base_str{nearest_base_info->first};
const auto base_entry{this->locations_.find(
{SchemaReferenceType::Static, nearest_base_str})};
if (base_entry != this->locations_.cend()) {
hoisted_base_view = base_entry->first.second;
hoisted_base_dialect = base_entry->second.base_dialect;
} else {
hoisted_base_view = nearest_base_info->first;
hoisted_base_dialect = root_base_dialect.value();
}
}

WeakPointer cached_base{};
for (const auto &base : every_base_result) {
Expand All @@ -769,39 +809,39 @@ auto SchemaFrame::analyse(const JSON &root, const SchemaWalker &walker,
this->locations_.contains({SchemaReferenceType::Static, result});

if (!contains) {
const auto nearest_bases{
find_nearest_bases(base_uris, pointer_weak,
std::optional<std::string_view>{base.first})};
assert(!nearest_bases.first.empty());
const auto &current_base{nearest_bases.first.front()};

const auto base_entry{this->locations_.find(
{SchemaReferenceType::Static, current_base})};

const std::string_view base_view{
base_entry != this->locations_.cend()
? std::string_view{base_entry->first.second}
: std::string_view{current_base}};

const sourcemeta::core::SchemaBaseDialect current_base_dialect{
base_entry != this->locations_.cend()
? base_entry->second.base_dialect
: root_base_dialect.value()};

const auto subschema{subschemas.find(pointer_weak)};
if (subschema != subschemas.cend()) {
std::string_view base_view;
sourcemeta::core::SchemaBaseDialect current_base_dialect;

if (nearest_base_info.has_value()) {
base_view = hoisted_base_view;
current_base_dialect = hoisted_base_dialect;
} else {
const JSON::String current_base{base.first};
const auto base_entry{this->locations_.find(
{SchemaReferenceType::Static, current_base})};
if (base_entry != this->locations_.cend()) {
base_view = base_entry->first.second;
current_base_dialect = base_entry->second.base_dialect;
} else {
base_view = base.first;
current_base_dialect = root_base_dialect.value();
}
}

if (is_subschema) {
store(this->locations_, SchemaReferenceType::Static,
SchemaFrame::LocationType::Subschema, std::move(result),
base_view, pointer_weak, nearest_bases.second.size(),
base_view, pointer_weak, nearest_base_depth,
dialect_for_pointer, current_base_dialect,
subschema->second.parent, false, true);
subschema_it->second.parent, false, true);
} else {
store(this->locations_, SchemaReferenceType::Static,
SchemaFrame::LocationType::Pointer, std::move(result),
base_view, pointer_weak, nearest_bases.second.size(),
base_view, pointer_weak, nearest_base_depth,
dialect_for_pointer, current_base_dialect,
dialect_match.has_value() ? dialect_match->second
: empty_weak_pointer,
combined.dialect_match.has_value()
? combined.dialect_match->second
: empty_weak_pointer,
false, true);
}
}
Expand All @@ -817,7 +857,7 @@ auto SchemaFrame::analyse(const JSON &root, const SchemaWalker &walker,
for (const auto &entry : subschema_entries) {
const auto &common_pointer_weak{entry.common.pointer};
if (entry.common.subschema.get().is_object()) {
const auto nearest_bases{find_nearest_bases(
const auto nearest_bases{find_nearest_bases<JSON::String>(
base_uris, common_pointer_weak,
entry.id ? std::optional<std::string_view>{*entry.id}
: std::nullopt)};
Expand Down