From 329e561effcdc751f860d289ffc13aa5e1a66df1 Mon Sep 17 00:00:00 2001 From: Tony Wasserka Date: Wed, 3 Jun 2026 17:36:51 +0200 Subject: [PATCH] FEXOfflineCompiler: Add "process-all" verb This operation will take care of any pending code cache operations: * import new code maps from `CACHE_DIR/codemap/new` and process them to `CACHE_DIR/codemap/ready` * generate caches for updated code maps with new blocks * ensure caches already exist for all other code maps (and generate them if needed) --- Source/Tools/FEXOfflineCompiler/Main.cpp | 208 +++++++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/Source/Tools/FEXOfflineCompiler/Main.cpp b/Source/Tools/FEXOfflineCompiler/Main.cpp index 80688afb85..0d7ecacb18 100644 --- a/Source/Tools/FEXOfflineCompiler/Main.cpp +++ b/Source/Tools/FEXOfflineCompiler/Main.cpp @@ -87,6 +87,11 @@ struct std::hash { } }; +// Windows requires O_BINARY, whereas on Linux it's implicit +#ifndef O_BINARY +#define O_BINARY 0 +#endif + // Placeholder data to ensure the compile thread doesn't de-reference nullptr data static FEXCore::Core::CPUState::gdt_segment gdt[32] {}; @@ -314,6 +319,206 @@ static int GenerateCache(int argc, const char** argv) { return GeneratedCache ? 0 : 1; } +/** + * Writes aggregated code map data into a single code map file that is ready to be used for cache generation + */ +static void WriteNewCodeMap(const FEXCore::ExecutableFileInfo& File, const std::string& OutputName, const fextl::set& Blocks, + bool IsExecutable, const std::set& Dependencies) { + fmt::print("Writing {} blocks to {}\n", Blocks.size(), OutputName); + + struct CodeMapOpener : FEXCore::CodeMapOpener { + CodeMapOpener(const std::string& Filename) { + FD = open(Filename.c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_BINARY, 0644); + } + + int OpenCodeMapFile() override { + return FD; + } + + int FD; + }; + + CodeMapOpener CodeMapOpener(OutputName); + FEXCore::CodeMapWriter OutputCodeMap(CodeMapOpener, true); + if (IsExecutable) { + // List the main executable and all used libraries + OutputCodeMap.AppendSetMainExecutable(File); + + for (auto& Dependency : Dependencies) { + OutputCodeMap.AppendLibraryLoad(Dependency); + } + } else { + // List only the library itself + OutputCodeMap.AppendLibraryLoad(File); + } + + for (auto& Block : Blocks) { + OutputCodeMap.AppendBlock(FEXCore::ExecutableFileSectionInfo {File, 0}, Block); + } +} + +struct ParsedContentsAndDependencies { + fextl::string Filename; + fextl::set Blocks; + bool IsExecutable = false; + std::set Dependencies; +}; + +/** + * Discovers any pending code maps, parses their contents into a runtime data structure, and deletes them + */ +static std::map ImportPendingCodeMaps(const std::string& NewCodeMapDirectory) { + // TODO: Handle nomb code maps + std::map Result; + for (auto& Entry : std::filesystem::directory_iterator(NewCodeMapDirectory)) { + if (!Entry.is_regular_file()) { + continue; + } + const auto Name = Entry.path().filename().string(); + if (!Name.ends_with(".bin")) { + continue; + } + + if (std::filesystem::file_size(Entry.path()) == 0) { + fmt::println("Found zero-size code map {}, deleting", Name); + std::filesystem::remove(Entry.path()); + continue; + } + + fmt::print("Importing new code map {}\n", Name); + std::ifstream Incoming(Entry.path(), std::ios_base::binary); + std::set Dependencies; + std::optional ExecutableFileId; + for (auto& [FileId, Contents] : FEXCore::CodeMap::ParseCodeMap(Incoming)) { + auto& [Filename, Blocks, IsExecutable, _] = + Result.emplace(std::piecewise_construct, std::forward_as_tuple(FileId), std::tuple {}).first->second; + Filename = std::move(Contents.Filename); + Blocks.merge(std::move(Contents.Blocks)); + IsExecutable = Contents.IsExecutable; + if (IsExecutable) { + LOGMAN_THROW_A_FMT(!ExecutableFileId, "Expected a unique executable identifiers per code map"); + ExecutableFileId = FileId; + } else { + Dependencies.insert(FileId); + } + } + + // Every imported code map should have had exactly one executable marker + LOGMAN_THROW_A_FMT(ExecutableFileId, "Could not find an executable identifer in the code map"); + Result.at(*ExecutableFileId).Dependencies = std::move(Dependencies); + + // Delete imported code map + Incoming.close(); + std::filesystem::remove(Entry.path()); + } + + return Result; +} + +/** + * Checks and processes new code maps generated by FEX + * + * Processed code maps are merged into the reference ("ready") code maps + */ +static void AggregateCodeMaps(const std::string& NewCodeMapDirectory, const std::string& ReadyCodeMapDirectory) { + auto IncomingCodeMap = ImportPendingCodeMaps(NewCodeMapDirectory); + + for (auto& [FileId, Contents] : IncomingCodeMap) { + // For each referenced binary, add the newly referenced offsets to that binary's reference code map + const FEXCore::ExecutableFileInfo File {nullptr, FileId, Contents.Filename}; + const auto BinaryName = std::string {FEXCore::CodeMap::GetBaseFilename(File, false)}; + auto OutputName = fmt::format("{}/{}", ReadyCodeMapDirectory, BinaryName); + + if (auto ReferenceCodeMap = std::ifstream(OutputName, std::ios_base::binary)) { + auto PreviousBlocks = FEXCore::CodeMap::ParseCodeMap(ReferenceCodeMap).at(File.FileId).Blocks; + auto NumPreviousBlocks = PreviousBlocks.size(); + Contents.Blocks.merge(std::move(PreviousBlocks)); + if (Contents.Blocks.size() == NumPreviousBlocks) { + // No new blocks => skip updating + continue; + } else { + fmt::println(" Found {} new blocks ({} total) in code map {} for {}", Contents.Blocks.size() - NumPreviousBlocks, + Contents.Blocks.size(), BinaryName, File.Filename); + } + } + + // Update code map + std::set Dependencies; + for (auto& Dependency : Contents.Dependencies) { + Dependencies.emplace(nullptr, Dependency, IncomingCodeMap.at(Dependency).Filename); + } + WriteNewCodeMap(File, OutputName, Contents.Blocks, Contents.IsExecutable, Dependencies); + } +} + +static int ProcessAll() { + const auto CacheDirectory = FEX::Config::GetCacheDirectory(); + const std::string NewCodeMapDirectory = fmt::format("{}codemap/new", CacheDirectory); + const std::string ReadyCodeMapDirectory = fmt::format("{}codemap/ready", CacheDirectory); + + // Import new code maps and aggregate them into ready code maps + std::filesystem::create_directories(ReadyCodeMapDirectory); + AggregateCodeMaps(NewCodeMapDirectory, ReadyCodeMapDirectory); + + // Generate caches + fextl::string OutDir = CacheDirectory + "cache/"; + std::filesystem::create_directories(OutDir); + + // Iterate over all executables (.exe). + // These determine the emulator configuration to use when compiling dependencies. + for (auto& Entry : std::filesystem::directory_iterator(ReadyCodeMapDirectory)) { + std::ifstream CodeMap(Entry.path(), std::ios_base::binary); + auto Parsed = FEXCore::CodeMap::ParseCodeMap(CodeMap); + auto ExecutableIt = std::ranges::find_if(Parsed, [](const auto& Entry) { return Entry.second.IsExecutable; }); + if (ExecutableIt == Parsed.end()) { + // Skip libraries; they're only processed as dependencies of a main executable + continue; + } + + fmt::println("\nChecking caches for executable {}", ExecutableIt->second.Filename); + + // TODO: Compute the cache config id from the active FEX configuration + uint64_t CodeCacheConfigId = 0; + + auto GetCacheFilename = [&](const FEXCore::ExecutableFileInfo& File) { + return fmt::format("{}{}-{:016x}", OutDir, FEXCore::CodeMap::GetBaseFilename(File, false), CodeCacheConfigId); + }; + + // Check the main binary and all of its dependencies + for (auto& [FileId, Contents] : Parsed) { + const FEXCore::ExecutableFileInfo File {nullptr, FileId, Contents.Filename}; + std::error_code ec; + const auto BinaryName = FEXCore::CodeMap::GetBaseFilename(File, false); + const auto MergedCodeMapFilename = fmt::format("{}/{}", ReadyCodeMapDirectory, BinaryName); + const auto LastCodeMapUpdate = std::filesystem::last_write_time(MergedCodeMapFilename, ec); + if (ec) { + // No reference code map exists for this dependency yet, so there's nothing to generate a cache from + continue; + } + + if (std::filesystem::last_write_time(GetCacheFilename(File), ec) > LastCodeMapUpdate && !ec) { + fmt::println(" Cache up to date: {}", BinaryName); + continue; + } + + // TODO: Also check for matching FEX version from cache header + + fmt::println(" {} cache: {}", ec ? "Generating" : "Updating outdated", BinaryName); + + // Defer to GenerateCache + const auto FileIdArg = fmt::format("{:016x}", FileId); + std::vector GenerateArgs { + "generate", "--fileid", FileIdArg.c_str(), "--outdir", OutDir.c_str(), MergedCodeMapFilename.c_str(), + }; + if (GenerateCache(GenerateArgs.size(), GenerateArgs.data()) != 0) { + fmt::println("ERROR: Cache generation failed for {}", BinaryName); + } + } + } + + return 0; +} + int main(int argc, char** argv) { LogMan::Throw::InstallHandler(AssertHandler); LogMan::Msg::InstallHandler(MsgHandler); @@ -324,10 +529,13 @@ int main(int argc, char** argv) { if (argc >= 2 && argv[1] == std::string_view {"generate"}) { return GenerateCache(argc - 1, Args.data()); + } else if (argc >= 2 && argv[1] == std::string_view {"process-all"}) { + return ProcessAll(); } else { fmt::print("Usage: {} \n\n", basename(argv[0])); fmt::print("Commands:\n"); fmt::print(" generate\tTrigger cache generation from combined code map\n"); + fmt::print(" process-all\tProcess all new code maps and update all caches\n"); return EXIT_FAILURE; } }