Skip to content

Migrate ExistingRustType AST importer to populate protos directly. #1521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
copybara-service wants to merge 1 commit into from
Closed

Migrate ExistingRustType AST importer to populate protos directly. #1521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions rs_bindings_from_cc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ cc_library(
deps = [
"cc_ir",
":bazel_types",
":ir_cc_proto",
"//lifetime_annotations",
"//lifetime_annotations:type_lifetimes",
"@abseil-cpp//absl/container:flat_hash_map",
Expand Down Expand Up @@ -323,6 +324,7 @@ cc_library(
":bazel_types",
":cc_ir",
":decl_importer",
":ir_cc_proto",
":recording_diagnostic_consumer",
":type_map",
"//common:annotation_reader",
Expand Down
66 changes: 65 additions & 1 deletion rs_bindings_from_cc/decl_importer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "lifetime_annotations/type_lifetimes.h"
#include "rs_bindings_from_cc/bazel_types.h"
#include "rs_bindings_from_cc/ir.h"
#include "rs_bindings_from_cc/ir.pb.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclBase.h"
#include "clang/AST/DeclTemplate.h"
Expand All @@ -34,6 +35,8 @@

namespace crubit {

namespace ir_proto = rs_bindings_from_cc::ir_proto::flat;

// Top-level parameters as well as return value of an importer invocation.
class Invocation {
public:
Expand Down Expand Up @@ -88,6 +91,7 @@ class Invocation {
const std::optional<absl::flat_hash_set<std::string>> do_not_bind_allowlist_;

// The main output of the import process
ir_proto::IRProto ir_proto_;
IR ir_;

// Transient map of top level items used to build the tree.
Expand Down Expand Up @@ -180,9 +184,37 @@ class ImportContext {
/*is_hard_error=*/false);
}

virtual std::unique_ptr<ir_proto::Item> ImportUnsupportedItemToProto(
const clang::Decl& decl, std::optional<UnsupportedItem::Path> path,
std::vector<FormattedError> errors, bool is_hard_error) {
IR::Item legacy_item = ImportUnsupportedItem(
decl, std::move(path), std::move(errors), is_hard_error);
auto proto_item = std::make_unique<ir_proto::Item>();
*proto_item = crubit::ToFlatProto(legacy_item);
return proto_item;
}

std::unique_ptr<ir_proto::Item> ImportUnsupportedItemToProto(
const clang::Decl& decl, std::optional<UnsupportedItem::Path> path,
std::vector<FormattedError> errors) {
return ImportUnsupportedItemToProto(decl, std::move(path),
std::move(errors),
/*is_hard_error=*/false);
}

std::unique_ptr<ir_proto::Item> HardErrorToProto(const clang::Decl& decl,
FormattedError error) {
return ImportUnsupportedItemToProto(decl, std::nullopt, {std::move(error)},
/*is_hard_error=*/true);
}

// Imports a decl and creates an IR item (or error messages). This allows
// importers to recursively delegate to other importers.
// Does not use or update the cache.
// TODO(deprecate-cpp-ir): Add virtual absl::StatusOr<flat_proto::Item*>
// ImportDeclToProto(clang::Decl* decl, bool must_bind) here (CL 1).
virtual absl::StatusOr<std::unique_ptr<ir_proto::Item>> ImportDeclToProto(
clang::Decl* decl, bool must_bind) = 0;
virtual std::optional<IR::Item> ImportDecl(clang::Decl* decl) = 0;

// Returns the Item of a Decl, importing it first if necessary.
Expand Down Expand Up @@ -383,6 +415,19 @@ class DeclImporter {
// be attempted, return UnsupportedItem.
virtual std::optional<IR::Item> ImportDecl(clang::Decl*, bool must_bind) = 0;

// Converts a decl to a proto IR item on the heap. Default
// implementation falls back to ImportDecl and converts via ToFlatProto.
virtual absl::StatusOr<std::unique_ptr<ir_proto::Item>> ImportDeclToProto(
clang::Decl* decl, bool must_bind) {
std::optional<IR::Item> legacy_item = ImportDecl(decl, must_bind);
if (!legacy_item.has_value()) {
return nullptr;
}
auto proto_item = std::make_unique<ir_proto::Item>();
*proto_item = crubit::ToFlatProto(*legacy_item);
return proto_item;
}

protected:
ImportContext& ictx_;
};
Expand All @@ -402,7 +447,26 @@ class DeclImporterBase : public DeclImporter {
must_bind_ = must_bind;
return Import(typed_decl);
}
virtual std::optional<IR::Item> Import(D*) = 0;
// TODO(b/532184858): Remove Import once all importers are migrated.
virtual std::optional<IR::Item> Import(D*) { return std::nullopt; }

absl::StatusOr<std::unique_ptr<ir_proto::Item>> ImportDeclToProto(
clang::Decl* decl, bool must_bind) override {
auto* typed_decl = clang::dyn_cast<D>(decl);
if (typed_decl == nullptr) return nullptr;
must_bind_ = must_bind;
return ImportToProto(typed_decl);
}
// TODO(b/532184858): Remove ToFlatProto fallback once all importers override
// this function.
virtual absl::StatusOr<std::unique_ptr<ir_proto::Item>> ImportToProto(
D* decl) {
std::optional<IR::Item> legacy_item = Import(decl);
if (!legacy_item.has_value()) return nullptr;
auto proto_item = std::make_unique<ir_proto::Item>();
*proto_item = crubit::ToFlatProto(*legacy_item);
return proto_item;
}

// A property of the current decl being imported.
// This is used to avoid re-parsing the annotation.
Expand Down
156 changes: 133 additions & 23 deletions rs_bindings_from_cc/importer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@
#include "llvm/Support/raw_ostream.h"

namespace crubit {

namespace ir_proto = rs_bindings_from_cc::ir_proto::flat;

namespace {

constexpr absl::string_view kTypeStatusPayloadUrl =
Expand Down Expand Up @@ -636,8 +639,8 @@ ItemId Importer::GenerateItemId(const clang::Decl* decl) const {

bool Importer::IsUnsupportedAndAlien(ItemId item_id) const {
auto it = import_cache_.find(reinterpret_cast<clang::Decl*>(item_id.value()));
return it != import_cache_.end() && it->second.has_value() &&
std::holds_alternative<UnsupportedItem>(*it->second) &&
return it != import_cache_.end() && it->second.legacy_item.has_value() &&
std::holds_alternative<UnsupportedItem>(*it->second.legacy_item) &&
!IsFromCurrentTarget(it->first);
}

Expand Down Expand Up @@ -942,14 +945,15 @@ void Importer::Import(clang::TranslationUnitDecl* translation_unit_decl) {
// class A { class B; }; // declares A::B
// class A::B { ... }; // defines A::B
std::vector<std::pair<SourceOrderKey, const clang::Decl*>> ordered_children;
for (const auto& [decl, item] : import_cache_) {
if (!item.has_value()) continue;
for (const auto& [decl, entry] : import_cache_) {
if (!entry.legacy_item.has_value()) continue;
if (auto* parent_record_decl =
llvm::dyn_cast<clang::CXXRecordDecl>(decl->getDeclContext())) {
auto parent_it = import_cache_.find(parent_record_decl);
if (parent_it != import_cache_.end() && parent_it->second.has_value()) {
if (parent_it != import_cache_.end() &&
parent_it->second.legacy_item.has_value()) {
if (auto* parent_item =
std::get_if<Record>(&(parent_it->second.value()))) {
std::get_if<Record>(&(parent_it->second.legacy_item.value()))) {
ordered_children.push_back({GetSourceOrderKey(decl), decl});
}
}
Expand All @@ -963,7 +967,8 @@ void Importer::Import(clang::TranslationUnitDecl* translation_unit_decl) {
auto* parent_record_decl =
llvm::dyn_cast<clang::CXXRecordDecl>(decl->getDeclContext());
auto parent_it = import_cache_.find(parent_record_decl);
auto* parent_item = std::get_if<Record>(&(parent_it->second.value()));
auto* parent_item =
std::get_if<Record>(&(parent_it->second.legacy_item.value()));

auto child_id = GenerateItemId(decl);
auto& parent_child_ids = invocation_.child_item_ids_[parent_item->id];
Expand All @@ -973,11 +978,12 @@ void Importer::Import(clang::TranslationUnitDecl* translation_unit_decl) {
}
}

for (const auto& [decl, item] : import_cache_) {
if (!item.has_value() || IsUnsupportedAndAlien(GenerateItemId(decl))) {
for (const auto& [decl, entry] : import_cache_) {
if (!entry.legacy_item.has_value() ||
IsUnsupportedAndAlien(GenerateItemId(decl))) {
continue;
}
ordered_items.push_back({GetSourceOrderKey(decl), *item});
ordered_items.push_back({GetSourceOrderKey(decl), *entry.legacy_item});
}

llvm::stable_sort(ordered_items, SourceLocationComparator(sm));
Expand Down Expand Up @@ -1018,7 +1024,10 @@ void Importer::ImportDeclsFromDeclContext(

std::optional<IR::Item> Importer::GetDeclItem(clang::Decl* decl) {
if (auto it = import_cache_.find(decl); it != import_cache_.end()) {
return it->second;
if (it->second.status == ItemCacheEntry::Status::kInProgress) {
return std::nullopt;
}
return it->second.legacy_item;
}
// Here, we need to be careful. Recursive imports break cycles as follows:
// an item which may, in the process of being imported, then import itself,
Expand Down Expand Up @@ -1054,8 +1063,14 @@ std::optional<IR::Item> Importer::GetDeclItem(clang::Decl* decl) {
// Note: insert_or_assign, not insert, in case a record, so as to overwrite
// any null entries introduced by cycles.

std::optional<IR::Item> result = ImportDecl(decl);
auto [it, inserted] = import_cache_.try_emplace(decl, result);
ItemId id = GenerateItemId(decl);
auto [it, inserted] = import_cache_.try_emplace(
decl, ItemCacheEntry{
.status = ItemCacheEntry::Status::kInProgress,
.id = id,
.legacy_item = std::nullopt,
.proto_item = nullptr,
});
if (!inserted) {
// TODO(jeanpierreda): Fix and promote to CHECK.
// At least one cycle occurs with Typedef, where a typedef will import
Expand All @@ -1066,14 +1081,31 @@ std::optional<IR::Item> Importer::GetDeclItem(clang::Decl* decl) {
//
// Alternatively, maybe it's sufficient to check that they're _equal_.
// It's not a bug at all to import it twice if it has no effect.
LOG_IF(INFO, !it->second.has_value())
LOG_IF(INFO, !it->second.legacy_item.has_value())
<< "re-entrant import discovered, where the re-entrant import had a "
"non-null value."
<< "\n trying to import a " << decl->getDeclKindName()
<< "\n present entry: " << ItemToString(it->second)
<< "\n was going to be inserted: " << ItemToString(result);
it->second = result;
<< "\n trying to import a " << decl->getDeclKindName();
}

std::optional<IR::Item> result = ImportDecl(decl);

std::unique_ptr<ir_proto::Item> proto_item = nullptr;
if (result.has_value()) {
proto_item = std::make_unique<ir_proto::Item>();
*proto_item = crubit::ToFlatProto(*result);
}

ItemCacheEntry::Status entry_status = result.has_value()
? ItemCacheEntry::Status::kCompleted
: ItemCacheEntry::Status::kFailed;

import_cache_[decl] = ItemCacheEntry{
.status = entry_status,
.id = id,
.legacy_item = result,
.proto_item = std::move(proto_item),
};

if (auto* record_decl = clang::dyn_cast<clang::CXXRecordDecl>(decl)) {
// TODO(forster): Should we even visit the nested decl if we couldn't
// import the parent? For now we have tests that check that we generate
Expand Down Expand Up @@ -1136,9 +1168,10 @@ std::optional<IR::Item> Importer::ImportDecl(clang::Decl* decl) {

if (IsTransitivelyInPrivate(decl)) {
if (*must_bind) {
return HardError(*decl,
FormattedError::Static(
"Private declarations cannot receive bindings"));
return HardError(
*decl,
FormattedError::Static("Items in private sections or classes are not "
"supported, but marked with must_bind"));
}
return std::nullopt;
}
Expand Down Expand Up @@ -1202,11 +1235,72 @@ std::optional<IR::Item> Importer::ImportDecl(clang::Decl* decl) {
return std::nullopt;
}

absl::StatusOr<std::unique_ptr<ir_proto::Item>> Importer::ImportDeclToProto(
clang::Decl* decl, bool must_bind) {
if (IsTransitivelyInPrivate(decl)) {
if (must_bind) {
return HardErrorToProto(
*decl,
FormattedError::Static("Items in private sections or classes are not "
"supported, but marked with must_bind"));
}
return nullptr;
}

const absl::StatusOr<bool> do_not_bind =
HasAnnotationWithoutArgs(*decl, "crubit_do_not_bind");
if (!do_not_bind.ok()) {
return HardErrorToProto(*decl,
FormattedError::FromStatus(do_not_bind.status()));
}
if (*do_not_bind) {
if (must_bind) {
return HardErrorToProto(
*decl, FormattedError::Static("Conflicting CRUBIT_MUST_BIND and "
"CRUBIT_DO_NOT_BIND annotations"));
}
const std::optional<absl::flat_hash_set<std::string>>&
do_not_bind_allowlist = invocation_.do_not_bind_allowlist_;
const clang::NamedDecl* named_decl =
clang::dyn_cast<clang::NamedDecl>(decl);
if (named_decl && !clang::isa<clang::FunctionDecl>(decl) &&
do_not_bind_allowlist.has_value()) {
std::string decl_name = named_decl->getQualifiedNameAsString();
if (!do_not_bind_allowlist->contains(decl_name)) {
return HardErrorToProto(
*decl, FormattedError::PrefixedStrCat(
"CRUBIT_DO_NOT_BIND annotation on non-allowlisted decl",
std::move(decl_name),
"\nOmitted bindings must be pre-registered using "
"`do_not_bind_allowlist`"));
}
}
return nullptr;
}

for (auto& decl_importer : decl_importers_) {
CRUBIT_ASSIGN_OR_RETURN(std::unique_ptr<ir_proto::Item> item,
decl_importer->ImportDeclToProto(decl, must_bind));
if (item != nullptr) {
return item;
}
}

if (must_bind) {
return HardErrorToProto(
*decl,
FormattedError::Static(
"No importer found for decl with CRUBIT_MUST_BIND annotation"));
}

return nullptr;
}

std::optional<IR::Item> Importer::GetImportedItem(
const clang::Decl* decl) const {
auto it = import_cache_.find(decl);
if (it != import_cache_.end()) {
return it->second;
return it->second.legacy_item;
}
return std::nullopt;
}
Expand Down Expand Up @@ -1377,7 +1471,7 @@ bool Importer::RefersToOwnedDefinitionImpl(
}

bool Importer::IsFromProtoTarget(const clang::Decl& decl) const {
// TODO(b/b/441343672): This is probably not a good way to detect if something
// TODO(b/441343672): This is probably not a good way to detect if something
// is from a proto target, and we should do something more durable.
clang::SourceManager& source_manager = ctx_.getSourceManager();
std::optional<llvm::StringRef> filename =
Expand Down Expand Up @@ -1608,6 +1702,22 @@ IR::Item Importer::HardError(const clang::Decl& decl, FormattedError error) {
/*is_hard_error=*/true);
}

std::unique_ptr<ir_proto::Item> Importer::HardErrorToProto(
const clang::Decl& decl, FormattedError error) {
return ImportUnsupportedItemToProto(decl, std::nullopt, {std::move(error)},
/*is_hard_error=*/true);
}

std::unique_ptr<ir_proto::Item> Importer::ImportUnsupportedItemToProto(
const clang::Decl& original_decl, std::optional<UnsupportedItem::Path> path,
std::vector<FormattedError> errors, bool is_hard_error) {
IR::Item legacy_item = ImportUnsupportedItem(
original_decl, std::move(path), std::move(errors), is_hard_error);
auto proto_item = std::make_unique<ir_proto::Item>();
*proto_item = crubit::ToFlatProto(legacy_item);
return proto_item;
}

IR::Item Importer::ImportUnsupportedItem(
const clang::Decl& original_decl, std::optional<UnsupportedItem::Path> path,
std::vector<FormattedError> errors, bool is_hard_error) {
Expand Down
Loading