From be0eb9aeb867e4b6a405d303d72dd281c373a588 Mon Sep 17 00:00:00 2001 From: LakatosMark Date: Sun, 29 Mar 2026 15:20:09 +0200 Subject: [PATCH 1/2] Fix #820: Refactor C# parser to extract target DLLs from .csproj and populate build graph --- .../include/csharpparser/csharpparser.h | 7 +- plugins/csharp/parser/src/csharpparser.cpp | 70 +++++----- plugins/csharp/parser/src_csharp/Program.cs | 125 ++++++++++-------- 3 files changed, 112 insertions(+), 90 deletions(-) diff --git a/plugins/csharp/parser/include/csharpparser/csharpparser.h b/plugins/csharp/parser/include/csharpparser/csharpparser.h index 6435dcf53..41e33e482 100644 --- a/plugins/csharp/parser/include/csharpparser/csharpparser.h +++ b/plugins/csharp/parser/include/csharpparser/csharpparser.h @@ -32,9 +32,10 @@ class CsharpParser : public AbstractParser bool acceptProjectBuildPath(const std::string& buildDir_); bool parseProjectBuildPath( - const std::vector& path_, - const std::string& buildPath_); - void addSource(const std::string& filepath_, bool error_); + const std::vector& path_ //, + ); //const std::string& buildPath_ + //void addSource(const std::string& filepath_, bool error_); + void addSource(const std::string& filepath_, const std::string& targetDll_, bool error_); }; } // parser diff --git a/plugins/csharp/parser/src/csharpparser.cpp b/plugins/csharp/parser/src/csharpparser.cpp index 490c82f56..c916fde63 100644 --- a/plugins/csharp/parser/src/csharpparser.cpp +++ b/plugins/csharp/parser/src/csharpparser.cpp @@ -27,11 +27,7 @@ CsharpParser::CsharpParser(ParserContext& ctx_): AbstractParser(ctx_) { _threadNum = _ctx.options["jobs"].as(); } -/* -bool CsharpParser::acceptProjectBuildPath(const std::vector& path_) -{ - return path_.size() >= 2 && fs::is_directory(path_[0]) && fs::is_directory(path_[1]); -}*/ + bool CsharpParser::acceptProjectBuildPath(const std::string& buildPath_) { return fs::is_directory(buildPath_); @@ -42,17 +38,15 @@ bool CsharpParser::parse() bool success = true; std::vector paths = _ctx.options["input"].as>(); - std::string buildPath = _ctx.options["build-dir"].as(); - if (acceptProjectBuildPath(buildPath)) + if (!paths.empty()) { - LOG(debug) << "C# parser parse path: " << paths[0]; - LOG(debug) << "Parsed csharp project build path: " << buildPath; - success = success && parseProjectBuildPath(paths, buildPath); + LOG(debug) << "C# parser parse path: " << paths.size(); + success = success && parseProjectBuildPath(paths); } else { - LOG(error) << "Build path must be a directory!"; + LOG(error) << "No input directories provided for C# parser!"; success = false; } @@ -60,8 +54,8 @@ bool CsharpParser::parse() } bool CsharpParser::parseProjectBuildPath( - const std::vector& paths_, - const std::string& buildPath_) + const std::vector& paths_ //, + ) { namespace ch = std::chrono; std::future log; @@ -80,8 +74,6 @@ bool CsharpParser::parseProjectBuildPath( command.append("'"); command.append(_ctx.options["database"].as()); command.append("' '"); - command.append(buildPath_); - command.append("' '"); command.append(csharp_path.string()); command.append("' "); command.append(std::to_string(_ctx.options["jobs"].as())); @@ -108,22 +100,35 @@ bool CsharpParser::parseProjectBuildPath( std::string line; std::stringstream log_str(log.get()); - //LOG(warning) << log_str.str(); int countFull = 0, countPart = 0; - + while(std::getline(log_str, line, '\n')) { if (line[0] == '+' || line[0] == '-') { - addSource(line.substr(1), line[0] == '-'); - if (line[0] == '+') - { - countFull++; + std::string content = line.substr(1); // We cut off the +/- sign + + // Find the line (|) that separates the file and the DLL + size_t separatorPos = content.find('|'); + + if (separatorPos != std::string::npos) { + // If it exists, we split the text along | + std::string filepath = content.substr(0, separatorPos); + std::string targetDll = content.substr(separatorPos + 1); + + // We clean up the spaces from the beginning + filepath.erase(0, filepath.find_first_not_of(" \t")); + + addSource(filepath, targetDll, line[0] == '-'); } - else - { - countPart++; + else { + // Fallback if for some reason the DLL name was not sent by C# + content.erase(0, content.find_first_not_of(" \t")); + addSource(content, "Unknown.dll", line[0] == '-'); } + + if (line[0] == '+') { countFull++; } + else { countPart++; } } } @@ -138,12 +143,12 @@ bool CsharpParser::parseProjectBuildPath( return result == 0; } -void CsharpParser::addSource(const std::string& filepath_, bool error_) +void CsharpParser::addSource(const std::string& filepath_, const std::string& targetDll_, bool error_) { util::OdbTransaction transaction(_ctx.db); model::BuildActionPtr buildAction(new model::BuildAction); - buildAction->command = " "; + buildAction->command = "dotnet build " + targetDll_; //buildAction->command = " "; buildAction->type = model::BuildAction::Compile; model::BuildSource buildSource; @@ -154,12 +159,21 @@ void CsharpParser::addSource(const std::string& filepath_, bool error_) buildSource.file->type = "CS"; buildSource.action = buildAction; + + model::BuildTarget buildTarget; + buildTarget.action = buildAction; + + buildTarget.file = _ctx.srcMgr.getFile(targetDll_); + buildTarget.file->type = "CS_DLL"; + _ctx.srcMgr.updateFile(*buildSource.file); + _ctx.srcMgr.updateFile(*buildTarget.file); _ctx.srcMgr.persistFiles(); transaction([&, this] { _ctx.db->persist(buildAction); _ctx.db->persist(buildSource); + _ctx.db->persist(buildTarget); //new!! }); } @@ -176,10 +190,6 @@ extern "C" { boost::program_options::options_description description("C# Plugin"); - description.add_options() - ("build-dir,b", po::value()->default_value("Build directory"), - "The build directory of the parsed project."); - return description; } diff --git a/plugins/csharp/parser/src_csharp/Program.cs b/plugins/csharp/parser/src_csharp/Program.cs index a9b2b23a5..e07ca2302 100644 --- a/plugins/csharp/parser/src_csharp/Program.cs +++ b/plugins/csharp/parser/src_csharp/Program.cs @@ -9,14 +9,13 @@ using System.Collections.Generic; using System.Threading.Tasks; using CSharpParser.model; +using System.Xml.Linq; namespace CSharpParser { class Program { - //private readonly CsharpDbContext _context; private static List _rootDir; - private static string _buildDir = ""; private static string _buildDirBase = ""; private static string _connectionString = ""; @@ -28,11 +27,10 @@ static int Main(string[] args) try { _connectionString = args[0].Replace("'", ""); - _buildDir = args[1].Replace("'", ""); - _buildDirBase = args[2].Replace("'", ""); - threadNum = int.Parse(args[3]); + _buildDirBase = args[1].Replace("'", ""); //indexes + threadNum = int.Parse(args[2]); - for (int i = 4; i < args.Length; ++i) + for (int i = 3; i < args.Length; ++i) { _rootDir.Add(args[i].Replace("'", "")); } @@ -42,44 +40,6 @@ static int Main(string[] args) WriteLine("Error in parsing command!"); return 1; } - /*if (args.Length < 3) - { - WriteLine("Missing command-line arguments in CSharpParser!"); - return 1; - } - else if (args.Length == 3) - { - _connectionString = args[0].Replace("'", ""); - _rootDir = args[1].Replace("'", ""); - _buildDir = args[2].Replace("'", ""); - } - else if (args.Length == 4) - { - _connectionString = args[0].Replace("'", ""); - _rootDir = args[1].Replace("'", ""); - _buildDir = args[2].Replace("'", ""); - bool success = int.TryParse(args[3], out threadNum); - if (!success){ - WriteLine("Invalid threadnumber argument! Multithreaded parsing disabled!"); - } - } - else if (args.Length == 5) - { - _connectionString = args[0].Replace("'", ""); - _rootDir = args[1].Replace("'", ""); - _buildDir = args[2].Replace("'", ""); - _buildDirBase = args[3].Replace("'", ""); - bool success = int.TryParse(args[4], out threadNum); - if (!success) - { - WriteLine("Invalid threadnumber argument! Multithreaded parsing disabled!"); - } - } - else if (args.Length > 5) - { - WriteLine("Too many command-line arguments in CSharpParser!"); - return 1; - }*/ //Converting the connectionstring into entiy framwork style connectionstring string csharpConnectionString = transformConnectionString(); @@ -91,21 +51,67 @@ static int Main(string[] args) CsharpDbContext _context = new CsharpDbContext(options); _context.Database.Migrate(); + List allFiles = new List(); + // This dictionary will remember which file belongs to which DLL + Dictionary fileToTargetDll = new Dictionary(); + foreach (var p in _rootDir) { - Console.WriteLine(p); - allFiles.AddRange(GetSourceFilesFromDir(p, ".cs")); + // We find all .csproj files + var csprojFiles = Directory.GetFiles(p, "*.csproj", SearchOption.AllDirectories); + foreach (var csproj in csprojFiles) + { + string projectDir = Path.GetDirectoryName(csproj); + // Default DLL name based on project file name + string targetDll = Path.GetFileNameWithoutExtension(csproj) + ".dll"; + + // we try to read the real AssemblyName from the XML + try { + XDocument doc = XDocument.Load(csproj); + var assemblyNameNode = doc.Descendants("AssemblyName").FirstOrDefault(); + if (assemblyNameNode != null && !string.IsNullOrWhiteSpace(assemblyNameNode.Value)) + { + targetDll = assemblyNameNode.Value + ".dll"; + } + } catch { /* If we cannot read the XML, the default name will remain.*/ } + + // search for C# files belonging to the project (filtering out the garbage) + var csFiles = Directory.GetFiles(projectDir, "*.cs", SearchOption.AllDirectories) + .Where(f => !f.Contains("/obj/") && !f.Contains("\\obj\\") && + !f.Contains("/bin/") && !f.Contains("\\bin\\")); + + foreach (var cs in csFiles) + { + // if a file is not already in it (to avoid duplication) + if (!fileToTargetDll.ContainsKey(cs)) + { + fileToTargetDll[cs] = targetDll; + allFiles.Add(cs); + } + } + } } + allFiles = allFiles.Distinct().ToList(); foreach (var f in allFiles) { WriteLine(f); } - IEnumerable assemblies = GetSourceFilesFromDir(_buildDir, ".dll"); - IEnumerable assemblies_base = assemblies; - if (args.Length == 5) - assemblies_base = GetSourceFilesFromDir(_buildDirBase, ".dll"); + + + IEnumerable assemblies_base = GetSourceFilesFromDir(_buildDirBase, ".dll"); //loading basic dlls + + List assemblies = new List(); + foreach (var p in _rootDir) + { + // We search for all .dll files in all input directories + assemblies.AddRange(GetSourceFilesFromDir(p, ".dll")); + } + // Let's keep only one of each DLL based on the file name! + assemblies = assemblies.GroupBy(x => System.IO.Path.GetFileName(x)) + .Select(g => g.First()) + .ToList(); List trees = new List(); foreach (string file in allFiles) @@ -129,14 +135,14 @@ static int Main(string[] args) compilation = compilation.AddReferences(MetadataReference.CreateFromFile(file)); } - var runtask = ParalellRun(csharpConnectionString, threadNum, trees, compilation); + var runtask = ParalellRun(csharpConnectionString, threadNum, trees, compilation, fileToTargetDll); int ret = runtask.Result; return 0; } private static async Task ParalellRun(string csharpConnectionString, int threadNum, - List trees, CSharpCompilation compilation) + List trees, CSharpCompilation compilation, Dictionary fileToTargetDll) { var options = new DbContextOptionsBuilder() .UseNpgsql(csharpConnectionString) @@ -156,7 +162,7 @@ private static async Task ParalellRun(string csharpConnectionString, int th WriteLine(threadNum); for (int i = 0; i < maxThread; i++) { - ParsingTasks.Add(ParseTree(contextList[i],trees[i],compilation,i)); + ParsingTasks.Add(ParseTree(contextList[i],trees[i],compilation,i,fileToTargetDll)); } int nextTreeIndex = maxThread; @@ -169,7 +175,7 @@ private static async Task ParalellRun(string csharpConnectionString, int th if (nextTreeIndex < trees.Count) { ParsingTasks.Add(ParseTree(contextList[nextContextIndex], - trees[nextTreeIndex],compilation,nextContextIndex)); + trees[nextTreeIndex],compilation,nextContextIndex, fileToTargetDll)); ++nextTreeIndex; } } @@ -183,15 +189,20 @@ private static async Task ParalellRun(string csharpConnectionString, int th } private static async Task ParseTree(CsharpDbContext context, - SyntaxTree tree, CSharpCompilation compilation, int index) + SyntaxTree tree, CSharpCompilation compilation, int index, + Dictionary fileToTargetDll) { var ParsingTask = Task.Run(() => { WriteLine("ParallelRun " + tree.FilePath); SemanticModel model = compilation.GetSemanticModel(tree); var visitor = new AstVisitor(context, model, tree); - visitor.Visit(tree.GetCompilationUnitRoot()); - WriteLine((visitor.FullyParsed ? "+" : "-") + tree.FilePath); + visitor.Visit(tree.GetCompilationUnitRoot()); + + // Find the DLL name and append a | to the filename. + string target = fileToTargetDll.ContainsKey(tree.FilePath) ? fileToTargetDll[tree.FilePath] : "Unknown.dll"; + WriteLine((visitor.FullyParsed ? "+" : "-") + tree.FilePath + "|" + target); + return index; }); return await ParsingTask; From 11653b46f5e97d9093ae09a896f0bdad9aad835b Mon Sep 17 00:00:00 2001 From: LakatosMark Date: Thu, 30 Apr 2026 13:02:23 +0200 Subject: [PATCH 2/2] Refactor C# IPC communication to JSON and remove placeholder Unknown.dll logic --- .../include/csharpparser/csharpparser.h | 5 +- plugins/csharp/parser/src/csharpparser.cpp | 63 +++++++++---------- plugins/csharp/parser/src_csharp/Program.cs | 45 ++++--------- 3 files changed, 43 insertions(+), 70 deletions(-) diff --git a/plugins/csharp/parser/include/csharpparser/csharpparser.h b/plugins/csharp/parser/include/csharpparser/csharpparser.h index 41e33e482..61dae01d1 100644 --- a/plugins/csharp/parser/include/csharpparser/csharpparser.h +++ b/plugins/csharp/parser/include/csharpparser/csharpparser.h @@ -32,9 +32,8 @@ class CsharpParser : public AbstractParser bool acceptProjectBuildPath(const std::string& buildDir_); bool parseProjectBuildPath( - const std::vector& path_ //, - ); //const std::string& buildPath_ - //void addSource(const std::string& filepath_, bool error_); + const std::vector& path_ + ); void addSource(const std::string& filepath_, const std::string& targetDll_, bool error_); }; diff --git a/plugins/csharp/parser/src/csharpparser.cpp b/plugins/csharp/parser/src/csharpparser.cpp index c916fde63..28b4b409e 100644 --- a/plugins/csharp/parser/src/csharpparser.cpp +++ b/plugins/csharp/parser/src/csharpparser.cpp @@ -18,6 +18,9 @@ #include +#include +#include + namespace cc { namespace parser @@ -104,31 +107,34 @@ bool CsharpParser::parseProjectBuildPath( while(std::getline(log_str, line, '\n')) { - if (line[0] == '+' || line[0] == '-') + // Skip empty lines or non-JSON lines (like debug info "ParallelRun ...") + if (line.empty() || line[0] != '{') continue; + + try { - std::string content = line.substr(1); // We cut off the +/- sign - - // Find the line (|) that separates the file and the DLL - size_t separatorPos = content.find('|'); - - if (separatorPos != std::string::npos) { - // If it exists, we split the text along | - std::string filepath = content.substr(0, separatorPos); - std::string targetDll = content.substr(separatorPos + 1); - - // We clean up the spaces from the beginning - filepath.erase(0, filepath.find_first_not_of(" \t")); - - addSource(filepath, targetDll, line[0] == '-'); - } - else { - // Fallback if for some reason the DLL name was not sent by C# - content.erase(0, content.find_first_not_of(" \t")); - addSource(content, "Unknown.dll", line[0] == '-'); - } - - if (line[0] == '+') { countFull++; } + std::stringstream jsonStream(line); + boost::property_tree::ptree pt; + boost::property_tree::read_json(jsonStream, pt); + + bool fullyParsed = pt.get("fullyParsed"); + std::string filepath = pt.get("filePath"); + std::string targetDll = pt.get("targetDll"); + + // Check if it's an error based on 'fullyParsed' + bool isError = !fullyParsed; + + addSource(filepath, targetDll, isError); + + if (fullyParsed) { countFull++; } else { countPart++; } + } + catch (const boost::property_tree::json_parser::json_parser_error& e) + { + LOG(warning) << "Failed to parse JSON output from C# parser: " << e.what() << " | Line: " << line; + } + catch (const boost::property_tree::ptree_error& e) + { + LOG(warning) << "Missing expected JSON field from C# parser: " << e.what() << " | Line: " << line; } } @@ -148,7 +154,7 @@ void CsharpParser::addSource(const std::string& filepath_, const std::string& ta util::OdbTransaction transaction(_ctx.db); model::BuildActionPtr buildAction(new model::BuildAction); - buildAction->command = "dotnet build " + targetDll_; //buildAction->command = " "; + buildAction->command = "dotnet build "; buildAction->type = model::BuildAction::Compile; model::BuildSource buildSource; @@ -159,21 +165,12 @@ void CsharpParser::addSource(const std::string& filepath_, const std::string& ta buildSource.file->type = "CS"; buildSource.action = buildAction; - - model::BuildTarget buildTarget; - buildTarget.action = buildAction; - - buildTarget.file = _ctx.srcMgr.getFile(targetDll_); - buildTarget.file->type = "CS_DLL"; - _ctx.srcMgr.updateFile(*buildSource.file); - _ctx.srcMgr.updateFile(*buildTarget.file); _ctx.srcMgr.persistFiles(); transaction([&, this] { _ctx.db->persist(buildAction); _ctx.db->persist(buildSource); - _ctx.db->persist(buildTarget); //new!! }); } diff --git a/plugins/csharp/parser/src_csharp/Program.cs b/plugins/csharp/parser/src_csharp/Program.cs index e07ca2302..aefbe8618 100644 --- a/plugins/csharp/parser/src_csharp/Program.cs +++ b/plugins/csharp/parser/src_csharp/Program.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using CSharpParser.model; using System.Xml.Linq; +using System.Text.Json; namespace CSharpParser { @@ -58,39 +59,7 @@ static int Main(string[] args) foreach (var p in _rootDir) { - // We find all .csproj files - var csprojFiles = Directory.GetFiles(p, "*.csproj", SearchOption.AllDirectories); - foreach (var csproj in csprojFiles) - { - string projectDir = Path.GetDirectoryName(csproj); - // Default DLL name based on project file name - string targetDll = Path.GetFileNameWithoutExtension(csproj) + ".dll"; - - // we try to read the real AssemblyName from the XML - try { - XDocument doc = XDocument.Load(csproj); - var assemblyNameNode = doc.Descendants("AssemblyName").FirstOrDefault(); - if (assemblyNameNode != null && !string.IsNullOrWhiteSpace(assemblyNameNode.Value)) - { - targetDll = assemblyNameNode.Value + ".dll"; - } - } catch { /* If we cannot read the XML, the default name will remain.*/ } - - // search for C# files belonging to the project (filtering out the garbage) - var csFiles = Directory.GetFiles(projectDir, "*.cs", SearchOption.AllDirectories) - .Where(f => !f.Contains("/obj/") && !f.Contains("\\obj\\") && - !f.Contains("/bin/") && !f.Contains("\\bin\\")); - - foreach (var cs in csFiles) - { - // if a file is not already in it (to avoid duplication) - if (!fileToTargetDll.ContainsKey(cs)) - { - fileToTargetDll[cs] = targetDll; - allFiles.Add(cs); - } - } - } + allFiles.AddRange(GetSourceFilesFromDir(p, ".cs")); } allFiles = allFiles.Distinct().ToList(); @@ -201,7 +170,15 @@ private static async Task ParseTree(CsharpDbContext context, // Find the DLL name and append a | to the filename. string target = fileToTargetDll.ContainsKey(tree.FilePath) ? fileToTargetDll[tree.FilePath] : "Unknown.dll"; - WriteLine((visitor.FullyParsed ? "+" : "-") + tree.FilePath + "|" + target); + + var resultData = new + { + fullyParsed = visitor.FullyParsed, + filePath = tree.FilePath, + targetDll = target + }; + + WriteLine(JsonSerializer.Serialize(resultData)); return index; });