From 09bd9a3996e7b587a995401c0a7c6661bf857c8a Mon Sep 17 00:00:00 2001 From: Kezia Rijadi Date: Wed, 22 Jul 2026 08:21:44 -0700 Subject: [PATCH] Scaffold the AST import loop to instantiate and store protobuf items. Note that these (proto) items are still yielded from C++ structs to minimal initial diffs in per-type importers. The follow-up CLs will refactor per-type importers to create and populate these protos directly. PiperOrigin-RevId: 952132930 --- rs_bindings_from_cc/BUILD | 2 + rs_bindings_from_cc/decl_importer.h | 45 +++++++ rs_bindings_from_cc/importer.cc | 182 ++++++++++++++++++++++++---- rs_bindings_from_cc/importer.h | 23 +++- 4 files changed, 227 insertions(+), 25 deletions(-) diff --git a/rs_bindings_from_cc/BUILD b/rs_bindings_from_cc/BUILD index 2e218776c..ebfb49552 100644 --- a/rs_bindings_from_cc/BUILD +++ b/rs_bindings_from_cc/BUILD @@ -271,6 +271,7 @@ cc_library( deps = [ "cc_ir", ":bazel_types", + ":ir_cc_proto", "//lifetime_annotations", "//lifetime_annotations:type_lifetimes", "@abseil-cpp//absl/container:flat_hash_map", @@ -323,6 +324,7 @@ cc_library( ":bazel_types", ":cc_ir", ":decl_importer", + ":ir_cc_proto", ":recording_diagnostic_consumer", ":type_map", "//common:annotation_reader", diff --git a/rs_bindings_from_cc/decl_importer.h b/rs_bindings_from_cc/decl_importer.h index 61b36e373..b7e1f97c3 100644 --- a/rs_bindings_from_cc/decl_importer.h +++ b/rs_bindings_from_cc/decl_importer.h @@ -21,6 +21,7 @@ #include "lifetime_annotations/type_lifetimes.h" #include "rs_bindings_from_cc/bazel_types.h" #include "rs_bindings_from_cc/ir.h" +#include "rs_bindings_from_cc/ir.pb.h" #include "clang/AST/Decl.h" #include "clang/AST/DeclBase.h" #include "clang/AST/DeclTemplate.h" @@ -34,6 +35,8 @@ namespace crubit { +namespace ir_proto = rs_bindings_from_cc::ir_proto::flat; + // Top-level parameters as well as return value of an importer invocation. class Invocation { public: @@ -88,6 +91,7 @@ class Invocation { const std::optional> do_not_bind_allowlist_; // The main output of the import process + ir_proto::IRProto ir_proto_; IR ir_; // Transient map of top level items used to build the tree. @@ -180,9 +184,37 @@ class ImportContext { /*is_hard_error=*/false); } + virtual std::unique_ptr ImportUnsupportedItemToProto( + const clang::Decl& decl, std::optional path, + std::vector errors, bool is_hard_error) { + IR::Item legacy_item = ImportUnsupportedItem( + decl, std::move(path), std::move(errors), is_hard_error); + auto proto_item = std::make_unique(); + *proto_item = crubit::ToFlatProto(legacy_item); + return proto_item; + } + + std::unique_ptr ImportUnsupportedItemToProto( + const clang::Decl& decl, std::optional path, + std::vector errors) { + return ImportUnsupportedItemToProto(decl, std::move(path), + std::move(errors), + /*is_hard_error=*/false); + } + + std::unique_ptr HardErrorToProto(const clang::Decl& decl, + FormattedError error) { + return ImportUnsupportedItemToProto(decl, std::nullopt, {std::move(error)}, + /*is_hard_error=*/true); + } + // Imports a decl and creates an IR item (or error messages). This allows // importers to recursively delegate to other importers. // Does not use or update the cache. + // TODO(deprecate-cpp-ir): Add virtual absl::StatusOr + // ImportDeclToProto(clang::Decl* decl, bool must_bind) here (CL 1). + virtual absl::StatusOr> ImportDeclToProto( + clang::Decl* decl, bool must_bind) = 0; virtual std::optional ImportDecl(clang::Decl* decl) = 0; // Returns the Item of a Decl, importing it first if necessary. @@ -383,6 +415,19 @@ class DeclImporter { // be attempted, return UnsupportedItem. virtual std::optional ImportDecl(clang::Decl*, bool must_bind) = 0; + // Converts a decl to a proto IR item on the heap. Default + // implementation falls back to ImportDecl and converts via ToFlatProto. + virtual absl::StatusOr> ImportDeclToProto( + clang::Decl* decl, bool must_bind) { + std::optional legacy_item = ImportDecl(decl, must_bind); + if (!legacy_item.has_value()) { + return nullptr; + } + auto proto_item = std::make_unique(); + *proto_item = crubit::ToFlatProto(*legacy_item); + return proto_item; + } + protected: ImportContext& ictx_; }; diff --git a/rs_bindings_from_cc/importer.cc b/rs_bindings_from_cc/importer.cc index 36e5c4ecd..a9dca8536 100644 --- a/rs_bindings_from_cc/importer.cc +++ b/rs_bindings_from_cc/importer.cc @@ -87,6 +87,9 @@ #include "llvm/Support/raw_ostream.h" namespace crubit { + +namespace ir_proto = rs_bindings_from_cc::ir_proto::flat; + namespace { constexpr absl::string_view kTypeStatusPayloadUrl = @@ -636,8 +639,8 @@ ItemId Importer::GenerateItemId(const clang::Decl* decl) const { bool Importer::IsUnsupportedAndAlien(ItemId item_id) const { auto it = import_cache_.find(reinterpret_cast(item_id.value())); - return it != import_cache_.end() && it->second.has_value() && - std::holds_alternative(*it->second) && + return it != import_cache_.end() && it->second.legacy_item.has_value() && + std::holds_alternative(*it->second.legacy_item) && !IsFromCurrentTarget(it->first); } @@ -922,6 +925,26 @@ bool Importer::IsAlwaysInstantiate( return false; } +void SetMustBindItem(ir_proto::Item& item) { + if (item.has_record()) { + item.mutable_record()->set_must_bind(true); + } else if (item.has_func()) { + item.mutable_func()->set_must_bind(true); + } else if (item.has_enum_decl()) { + item.mutable_enum_decl()->set_must_bind(true); + } else if (item.has_type_alias()) { + item.mutable_type_alias()->set_must_bind(true); + } else if (item.has_comment()) { + item.mutable_comment()->set_must_bind(true); + } else if (item.has_unsupported_item()) { + item.mutable_unsupported_item()->set_must_bind(true); + } else if (item.has_namespace_decl()) { + item.mutable_namespace_decl()->set_must_bind(true); + } else if (item.has_use_mod()) { + item.mutable_use_mod()->set_must_bind(true); + } +} + void Importer::Import(clang::TranslationUnitDecl* translation_unit_decl) { FindAlwaysInstantiateSpecs(translation_unit_decl); ImportFreeComments(); @@ -942,14 +965,15 @@ void Importer::Import(clang::TranslationUnitDecl* translation_unit_decl) { // class A { class B; }; // declares A::B // class A::B { ... }; // defines A::B std::vector> ordered_children; - for (const auto& [decl, item] : import_cache_) { - if (!item.has_value()) continue; + for (const auto& [decl, entry] : import_cache_) { + if (!entry.legacy_item.has_value()) continue; if (auto* parent_record_decl = llvm::dyn_cast(decl->getDeclContext())) { auto parent_it = import_cache_.find(parent_record_decl); - if (parent_it != import_cache_.end() && parent_it->second.has_value()) { + if (parent_it != import_cache_.end() && + parent_it->second.legacy_item.has_value()) { if (auto* parent_item = - std::get_if(&(parent_it->second.value()))) { + std::get_if(&(parent_it->second.legacy_item.value()))) { ordered_children.push_back({GetSourceOrderKey(decl), decl}); } } @@ -963,7 +987,8 @@ void Importer::Import(clang::TranslationUnitDecl* translation_unit_decl) { auto* parent_record_decl = llvm::dyn_cast(decl->getDeclContext()); auto parent_it = import_cache_.find(parent_record_decl); - auto* parent_item = std::get_if(&(parent_it->second.value())); + auto* parent_item = + std::get_if(&(parent_it->second.legacy_item.value())); auto child_id = GenerateItemId(decl); auto& parent_child_ids = invocation_.child_item_ids_[parent_item->id]; @@ -973,11 +998,12 @@ void Importer::Import(clang::TranslationUnitDecl* translation_unit_decl) { } } - for (const auto& [decl, item] : import_cache_) { - if (!item.has_value() || IsUnsupportedAndAlien(GenerateItemId(decl))) { + for (const auto& [decl, entry] : import_cache_) { + if (!entry.legacy_item.has_value() || + IsUnsupportedAndAlien(GenerateItemId(decl))) { continue; } - ordered_items.push_back({GetSourceOrderKey(decl), *item}); + ordered_items.push_back({GetSourceOrderKey(decl), *entry.legacy_item}); } llvm::stable_sort(ordered_items, SourceLocationComparator(sm)); @@ -1018,7 +1044,10 @@ void Importer::ImportDeclsFromDeclContext( std::optional Importer::GetDeclItem(clang::Decl* decl) { if (auto it = import_cache_.find(decl); it != import_cache_.end()) { - return it->second; + if (it->second.status == ItemCacheEntry::Status::kInProgress) { + return std::nullopt; + } + return it->second.legacy_item; } // Here, we need to be careful. Recursive imports break cycles as follows: // an item which may, in the process of being imported, then import itself, @@ -1054,8 +1083,14 @@ std::optional Importer::GetDeclItem(clang::Decl* decl) { // Note: insert_or_assign, not insert, in case a record, so as to overwrite // any null entries introduced by cycles. - std::optional result = ImportDecl(decl); - auto [it, inserted] = import_cache_.try_emplace(decl, result); + ItemId id = GenerateItemId(decl); + auto [it, inserted] = import_cache_.try_emplace( + decl, ItemCacheEntry{ + .status = ItemCacheEntry::Status::kInProgress, + .id = id, + .legacy_item = std::nullopt, + .proto_item = nullptr, + }); if (!inserted) { // TODO(jeanpierreda): Fix and promote to CHECK. // At least one cycle occurs with Typedef, where a typedef will import @@ -1066,14 +1101,27 @@ std::optional Importer::GetDeclItem(clang::Decl* decl) { // // Alternatively, maybe it's sufficient to check that they're _equal_. // It's not a bug at all to import it twice if it has no effect. - LOG_IF(INFO, !it->second.has_value()) + LOG_IF(INFO, !it->second.legacy_item.has_value()) << "re-entrant import discovered, where the re-entrant import had a " "non-null value." - << "\n trying to import a " << decl->getDeclKindName() - << "\n present entry: " << ItemToString(it->second) - << "\n was going to be inserted: " << ItemToString(result); - it->second = result; + << "\n trying to import a " << decl->getDeclKindName(); } + + std::optional result = ImportDecl(decl); + + ItemCacheEntry::Status entry_status = result.has_value() + ? ItemCacheEntry::Status::kCompleted + : ItemCacheEntry::Status::kFailed; + + import_cache_[decl] = ItemCacheEntry{ + .status = entry_status, + .id = id, + .legacy_item = result, + .proto_item = result.has_value() ? std::make_unique( + crubit::ToFlatProto(*result)) + : nullptr, + }; + if (auto* record_decl = clang::dyn_cast(decl)) { // TODO(forster): Should we even visit the nested decl if we couldn't // import the parent? For now we have tests that check that we generate @@ -1136,9 +1184,10 @@ std::optional Importer::ImportDecl(clang::Decl* decl) { if (IsTransitivelyInPrivate(decl)) { if (*must_bind) { - return HardError(*decl, - FormattedError::Static( - "Private declarations cannot receive bindings")); + return HardError( + *decl, + FormattedError::Static("Items in private sections or classes are not " + "supported, but marked with must_bind")); } return std::nullopt; } @@ -1202,11 +1251,84 @@ std::optional Importer::ImportDecl(clang::Decl* decl) { return std::nullopt; } +absl::StatusOr> Importer::ImportDeclToProto( + clang::Decl* decl, bool must_bind) { + if (IsTransitivelyInPrivate(decl)) { + if (must_bind) { + return HardErrorToProto( + *decl, + FormattedError::Static("Items in private sections or classes are not " + "supported, but marked with must_bind")); + } + return nullptr; + } + + const absl::StatusOr do_not_bind = + HasAnnotationWithoutArgs(*decl, "crubit_do_not_bind"); + if (!do_not_bind.ok()) { + return HardErrorToProto(*decl, + FormattedError::FromStatus(do_not_bind.status())); + } + if (*do_not_bind) { + if (must_bind) { + return HardErrorToProto( + *decl, FormattedError::Static("Conflicting CRUBIT_MUST_BIND and " + "CRUBIT_DO_NOT_BIND annotations")); + } + const std::optional>& + do_not_bind_allowlist = invocation_.do_not_bind_allowlist_; + const clang::NamedDecl* named_decl = + clang::dyn_cast(decl); + if (named_decl && !clang::isa(decl) && + do_not_bind_allowlist.has_value()) { + std::string decl_name = named_decl->getQualifiedNameAsString(); + if (!do_not_bind_allowlist->contains(decl_name)) { + return HardErrorToProto( + *decl, FormattedError::PrefixedStrCat( + "CRUBIT_DO_NOT_BIND annotation on non-allowlisted decl", + std::move(decl_name), + "\nOmitted bindings must be pre-registered using " + "`do_not_bind_allowlist`")); + } + } + return nullptr; + } + + std::string unavailable_error; + if (decl->isUnavailable(&unavailable_error)) { + return ImportUnsupportedItemToProto( + *decl, std::nullopt, + {FormattedError::PrefixedStrCat("Decl is unavailable: ", + std::move(unavailable_error))}, + /*is_hard_error=*/must_bind); + } + + for (auto& decl_importer : decl_importers_) { + CRUBIT_ASSIGN_OR_RETURN(std::unique_ptr item, + decl_importer->ImportDeclToProto(decl, must_bind)); + if (item != nullptr) { + if (must_bind) { + SetMustBindItem(*item); + } + return item; + } + } + + if (must_bind) { + return HardErrorToProto( + *decl, + FormattedError::Static( + "No importer found for decl with CRUBIT_MUST_BIND annotation")); + } + + return nullptr; +} + std::optional Importer::GetImportedItem( const clang::Decl* decl) const { auto it = import_cache_.find(decl); if (it != import_cache_.end()) { - return it->second; + return it->second.legacy_item; } return std::nullopt; } @@ -1377,7 +1499,7 @@ bool Importer::RefersToOwnedDefinitionImpl( } bool Importer::IsFromProtoTarget(const clang::Decl& decl) const { - // TODO(b/b/441343672): This is probably not a good way to detect if something + // TODO(b/441343672): This is probably not a good way to detect if something // is from a proto target, and we should do something more durable. clang::SourceManager& source_manager = ctx_.getSourceManager(); std::optional filename = @@ -1608,6 +1730,20 @@ IR::Item Importer::HardError(const clang::Decl& decl, FormattedError error) { /*is_hard_error=*/true); } +std::unique_ptr Importer::HardErrorToProto( + const clang::Decl& decl, FormattedError error) { + return ImportUnsupportedItemToProto(decl, std::nullopt, {std::move(error)}, + /*is_hard_error=*/true); +} + +std::unique_ptr Importer::ImportUnsupportedItemToProto( + const clang::Decl& original_decl, std::optional path, + std::vector errors, bool is_hard_error) { + IR::Item legacy_item = ImportUnsupportedItem( + original_decl, std::move(path), std::move(errors), is_hard_error); + return std::make_unique(crubit::ToFlatProto(legacy_item)); +} + IR::Item Importer::ImportUnsupportedItem( const clang::Decl& original_decl, std::optional path, std::vector errors, bool is_hard_error) { diff --git a/rs_bindings_from_cc/importer.h b/rs_bindings_from_cc/importer.h index f326f9a2e..959faafb2 100644 --- a/rs_bindings_from_cc/importer.h +++ b/rs_bindings_from_cc/importer.h @@ -21,6 +21,7 @@ #include "rs_bindings_from_cc/bazel_types.h" #include "rs_bindings_from_cc/decl_importer.h" #include "rs_bindings_from_cc/ir.h" +#include "rs_bindings_from_cc/ir.pb.h" #include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclTemplate.h" @@ -32,6 +33,18 @@ namespace crubit { +namespace ir_proto = rs_bindings_from_cc::ir_proto::flat; + +// Stateful entry to prevent re-entrant imports, and to track the underlying +// proto item we should store the AST node in. +struct ItemCacheEntry { + enum class Status { kInProgress, kCompleted, kFailed, kUnsupported }; + Status status = Status::kCompleted; + ItemId id = ItemId(0); + std::optional legacy_item; + std::unique_ptr proto_item; +}; + // Iterates over the AST created from the invocation's entry headers and // creates an intermediate representation of the import (`IR`) into the // invocation object. @@ -48,11 +61,18 @@ class Importer final : public ImportContext { void ImportDeclsFromDeclContext( const clang::DeclContext* decl_context) override; IR::Item HardError(const clang::Decl& decl, FormattedError error) override; + std::unique_ptr HardErrorToProto(const clang::Decl& decl, + FormattedError error); IR::Item ImportUnsupportedItem(const clang::Decl& decl, std::optional path, std::vector errors, bool is_hard_error) override; + std::unique_ptr ImportUnsupportedItemToProto( + const clang::Decl& decl, std::optional path, + std::vector errors, bool is_hard_error) override; std::optional ImportDecl(clang::Decl* decl) override; + absl::StatusOr> ImportDeclToProto( + clang::Decl* decl, bool must_bind) override; std::optional GetImportedItem( const clang::Decl* decl) const override; @@ -198,8 +218,7 @@ class Importer final : public ImportContext { // to successfully match a decl "wins", and no other importers are tried. std::vector> decl_importers_; std::unique_ptr mangler_; - absl::flat_hash_map> - import_cache_; + absl::flat_hash_map import_cache_; absl::flat_hash_set class_template_instantiations_; std::vector comments_;