Tweaks to ModResources helpers, Story.cs

.editorconfig

@@ -0,0 +1,11 @@
+# Remove the line below if you want to inherit .editorconfig settings from higher directories
+root = true
+
+# C# files
+[*.cs]
+
+#### Core EditorConfig Options ####
+
+# Indentation and spacing
+indent_size = 4
+indent_style = space

DebuggerFrontend/DebuggerFrontend.csproj

@@ -34,11 +34,11 @@
     <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="CommandLineArgumentsParser, Version=3.0.19.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\CommandLineArgumentsParser.3.0.19\lib\net452\CommandLineArgumentsParser.dll</HintPath>
+    <Reference Include="CommandLineArgumentsParser, Version=3.0.22.0, Culture=neutral, PublicKeyToken=2990a79b704d8378, processorArchitecture=MSIL">
+      <HintPath>..\packages\CommandLineArgumentsParser.3.0.22\lib\net452\CommandLineArgumentsParser.dll</HintPath>
     </Reference>
-    <Reference Include="Google.Protobuf, Version=3.6.1.0, Culture=neutral, PublicKeyToken=a7d26565bac4d604, processorArchitecture=MSIL">
-      <HintPath>..\packages\Google.Protobuf.3.6.1\lib\net45\Google.Protobuf.dll</HintPath>
+    <Reference Include="Google.Protobuf, Version=3.21.2.0, Culture=neutral, PublicKeyToken=a7d26565bac4d604, processorArchitecture=MSIL">
+      <HintPath>..\packages\Google.Protobuf.3.21.2\lib\net45\Google.Protobuf.dll</HintPath>
     </Reference>
     <Reference Include="Newtonsoft.Json, Version=13.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <HintPath>..\packages\Newtonsoft.Json.13.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
@@ -47,10 +47,22 @@
       <HintPath>..\External\gppg\binaries\QUT.ShiftReduceParser.dll</HintPath>
     </Reference>
     <Reference Include="System" />
+    <Reference Include="System.Buffers, Version=4.0.3.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Buffers.4.5.1\lib\net461\System.Buffers.dll</HintPath>
+    </Reference>
     <Reference Include="System.Core" />
     <Reference Include="System.Drawing" />
     <Reference Include="System.IO.Compression.FileSystem" />
+    <Reference Include="System.Memory, Version=4.0.1.2, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Memory.4.5.5\lib\net461\System.Memory.dll</HintPath>
+    </Reference>
     <Reference Include="System.Numerics" />
+    <Reference Include="System.Numerics.Vectors, Version=4.1.4.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Numerics.Vectors.4.5.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Runtime.CompilerServices.Unsafe, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Runtime.CompilerServices.Unsafe.6.0.0\lib\net461\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
+    </Reference>
     <Reference Include="System.Runtime.Serialization" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />

DebuggerFrontend/packages.config

@@ -1,6 +1,10 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="CommandLineArgumentsParser" version="3.0.19" targetFramework="net462" />
-  <package id="Google.Protobuf" version="3.15.0" targetFramework="net461" />
+  <package id="CommandLineArgumentsParser" version="3.0.22" targetFramework="net461" />
+  <package id="Google.Protobuf" version="3.21.2" targetFramework="net461" />
   <package id="Newtonsoft.Json" version="13.0.1" targetFramework="net461" />
+  <package id="System.Buffers" version="4.5.1" targetFramework="net461" />
+  <package id="System.Memory" version="4.5.5" targetFramework="net461" />
+  <package id="System.Numerics.Vectors" version="4.5.0" targetFramework="net461" />
+  <package id="System.Runtime.CompilerServices.Unsafe" version="6.0.0" targetFramework="net461" />
 </packages>

Divine/Divine.csproj

@@ -38,8 +38,8 @@
     <Reference Include="AlphaFS, Version=2.2.0.0, Culture=neutral, PublicKeyToken=4d31a58f7d7ad5c9, processorArchitecture=MSIL">
       <HintPath>..\packages\AlphaFS.2.2.6\lib\net452\AlphaFS.dll</HintPath>
     </Reference>
-    <Reference Include="CommandLineArgumentsParser, Version=3.0.19.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\CommandLineArgumentsParser.3.0.19\lib\net452\CommandLineArgumentsParser.dll</HintPath>
+    <Reference Include="CommandLineArgumentsParser, Version=3.0.22.0, Culture=neutral, PublicKeyToken=2990a79b704d8378, processorArchitecture=MSIL">
+      <HintPath>..\packages\CommandLineArgumentsParser.3.0.22\lib\net452\CommandLineArgumentsParser.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />

Divine/packages.config

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="AlphaFS" version="2.2.6" targetFramework="net462" />
-  <package id="CommandLineArgumentsParser" version="3.0.19" targetFramework="net462" />
+  <package id="CommandLineArgumentsParser" version="3.0.22" targetFramework="net461" />
 </packages>

LSLib/LS/Mods/ModResources.cs

@@ -1,10 +1,14 @@
-using LSLib.LS.Story.Compiler;
+#define DEBUG
+using LSLib.LS.Story.Compiler;
+
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
 
+using System.Diagnostics;
+
 namespace LSLib.LS
 {
     public class ModInfo
@@ -48,7 +52,7 @@ public class ModPathVisitor
         private static readonly Regex globalsRe = new Regex("^Mods/([^/]+)/Globals/.*/.*/.*\\.lsf$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
         private static readonly Regex levelObjectsRe = new Regex("^Mods/([^/]+)/Levels/.*/(Characters|Items|Triggers)/.*\\.lsf$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
         // Pattern for excluding subsequent parts of a multi-part archive
-        private static readonly Regex archivePartRe = new Regex("^(.*)_[1-9]\\.pak$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
+        private static readonly Regex archivePartRe = new Regex("^(.*)_\\d+(.pak)$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
 
         private readonly ModResources Resources;
 
@@ -210,55 +214,114 @@ private void DiscoverPackagedFile(AbstractFileInfo file)
 
         public void DiscoverPackage(string packagePath)
         {
-            var reader = new PackageReader(packagePath);
-            Resources.LoadedPackages.Add(reader);
-            var package = reader.Read();
+            try
+            {
+                var reader = new PackageReader(packagePath);
+                Resources.LoadedPackages.Add(reader);
+                var package = reader.Read();
 
-            foreach (var file in package.Files)
+                foreach (var file in package.Files)
+                {
+                    DiscoverPackagedFile(file);
+                }
+            }
+            catch(Exception ex)
             {
-                DiscoverPackagedFile(file);
+               Debug.WriteLine($"Error parsing package ({packagePath}):\n{ex}");
             }
         }
 
-        public void DiscoverBuiltinPackages(string gameDataPath)
+        // List of packages we won't ever load
+        // These packages don't contain any mod resources, but have a large
+        // file table that makes loading unneccessarily slow.
+        static HashSet<string> packageBlacklist = new HashSet<string>
+        {
+            "Assets.pak",
+            "Effects.pak",
+            "Engine.pak",
+            "EngineShaders.pak",
+            "Game.pak",
+            "GamePlatform.pak",
+            "Gustav_Textures.pak",
+            "Icons.pak",
+            "LowTex.pak",
+            "Materials.pak",
+            "Minimaps.pak",
+            "Models.pak",
+            "SharedSoundBanks.pak",
+            "SharedSounds.pak",
+            "Textures.pak",
+            "VirtualTextures.pak"
+        };
+
+        static bool IgnorePak(Alphaleonis.Win32.Filesystem.FileSystemEntryInfo f, IEnumerable<string> ignorePaks = null)
+        {
+            if (ignorePaks != null)
+            {
+                if (ignorePaks.Contains(f.FileName, StringComparer.OrdinalIgnoreCase))
+                {
+                    return true;
+                }
+                var m = archivePartRe.Match(f.FileName);
+                if (m.Success)
+                {
+                    var baseName = m.Groups[1].Value + m.Groups[2].Value;
+                    return ignorePaks.Contains(baseName, StringComparer.OrdinalIgnoreCase);
+                }
+            }
+            return false;
+        }
+
+        static bool IsPartialPak(KeyValuePair<string, string> kvp, Dictionary<string, string> paks)
         {
-            // List of packages we won't ever load
-            // These packages don't contain any mod resources, but have a large
-            // file table that makes loading unneccessarily slow.
-            HashSet<string> packageBlacklist = new HashSet<string>
+            var m = archivePartRe.Match(kvp.Key);
+            if (m.Success)
             {
-                "Assets.pak",
-                "Effects.pak",
-                "Engine.pak",
-                "EngineShaders.pak",
-                "Game.pak",
-                "GamePlatform.pak",
-                "Gustav_Textures.pak",
-                "Icons.pak",
-                "LowTex.pak",
-                "Materials.pak",
-                "Minimaps.pak",
-                "Models.pak",
-                "SharedSoundBanks.pak",
-                "SharedSounds.pak",
-                "Textures.pak",
-                "VirtualTextures.pak"
+                // Get the name without any numbers
+                var baseName = m.Groups[1].Value + m.Groups[2].Value;
+                // Check if this name is in the dictionary already. If it is, then this is a partial pak.
+                return paks.ContainsKey(baseName);
+            }
+            return false;
+        }
+
+        static IEnumerable<string> GetPackagePaths(string rootDirectory, bool recursive = false, IEnumerable<string> ignorePaks = null)
+        {
+            var opts = Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.Files;
+            if (recursive)
+            {
+                opts |= Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.Recursive;
+            }
+
+            var filters = new Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters
+            {
+                InclusionFilter = (f) => f.Extension.Equals(".pak", StringComparison.OrdinalIgnoreCase) && !IgnorePak(f, ignorePaks)
             };
+            var allPackages = Alphaleonis.Win32.Filesystem.Directory.EnumerateFiles(rootDirectory, opts, filters)
+                .ToDictionary(x => Alphaleonis.Win32.Filesystem.Path.GetFileName(x), x => x);
+
+            return allPackages.Where(p => !IsPartialPak(p, allPackages)).Select(kvp => kvp.Value).Distinct();
+        }
 
+        public void DiscoverBuiltinPackages(string gameDataPath)
+        {
             // Collect priority value from headers
             var packagePriorities = new List<Tuple<string, int>>();
 
-            foreach (var path in Directory.GetFiles(gameDataPath, "*.pak"))
+            var packagePaths = GetPackagePaths(gameDataPath, false, packageBlacklist).ToList();
+
+            foreach (var path in packagePaths)
             {
-                var baseName = Path.GetFileName(path);
-                if (!packageBlacklist.Contains(baseName)
-                    // Don't load 2nd, 3rd, ... parts of a multi-part archive
-                    && !archivePartRe.IsMatch(baseName))
+                try
                 {
                     var reader = new PackageReader(path, true);
                     var package = reader.Read();
                     packagePriorities.Add(new Tuple<string, int>(path, package.Metadata.Priority));
                 }
+                catch (Exception ex)
+                {
+                    Debug.WriteLine($"Error parsing builtin package:\n{ex}");
+                }
             }
 
             packagePriorities.Sort(
@@ -271,19 +334,15 @@ public void DiscoverBuiltinPackages(string gameDataPath)
             // Load non-patch packages first
             foreach (var package in packagePriorities)
             {
-                 DiscoverPackage(package.Item1);
+                DiscoverPackage(package.Item1);
             }
         }
 
         public void DiscoverUserPackages(string gameDataPath)
         {
-            foreach (var packagePath in Directory.GetFiles(gameDataPath, "*.pak"))
+            foreach (var packagePath in GetPackagePaths(gameDataPath, false))
             {
-                // Don't load 2nd, 3rd, ... parts of a multi-part archive
-                if (!archivePartRe.IsMatch(packagePath))
-                {
-                    DiscoverPackage(packagePath);
-                }
+                DiscoverPackage(packagePath);
             }
         }
 
@@ -407,7 +466,7 @@ public void DiscoverModDirectory(string modName, string modPath, string publicPa
                 }
             }
 
-            if (CollectStats)
+            if (CollectStats && Directory.Exists(publicPath))
             {
                 DiscoverModStats(modName, publicPath);
             }
@@ -454,4 +513,4 @@ public void Discover(String gameDataPath)
             DiscoverMods(gameDataPath);
         }
     }
-}
+}

LSLib/LS/PackageReader.cs

@@ -11,16 +11,41 @@ namespace LSLib.LS
 {
     public class NotAPackageException : Exception
     {
-        public NotAPackageException()
-        {
-        }
+        private string _fileName;
+        public string FileName => _fileName;
+
+        public NotAPackageException() { }
+
+        public NotAPackageException(string message) : base(message) { }
 
-        public NotAPackageException(string message) : base(message)
+        public NotAPackageException(string message, string fileName) : base(message)
         {
+            _fileName = fileName;
         }
 
-        public NotAPackageException(string message, Exception innerException) : base(message, innerException)
+        public NotAPackageException(string message, Exception innerException) : base(message, innerException) { }
+
+        public override string ToString()
         {
+            string s = GetType().FullName + ": " + Message;
+
+            if (_fileName != null && _fileName.Length != 0)
+            {
+                s += Environment.NewLine + "File: " + _fileName;
+            }
+
+            if (InnerException != null)
+            {
+                s = s + " ---> " + InnerException.ToString();
+            }
+
+            if (StackTrace != null)
+            {
+                s += Environment.NewLine + StackTrace;
+            }
+
+            return s;
+
         }
     }
 
@@ -72,7 +97,7 @@ private Package ReadPackageV7(FileStream mainStream, BinaryReader reader)
 
             if (_metadataOnly) return package;
 
-            OpenStreams(mainStream, (int) header.NumParts);
+            OpenStreams(mainStream, (int)header.NumParts);
             for (uint i = 0; i < header.NumFiles; i++)
             {
                 var entry = BinUtils.ReadStruct<FileEntry7>(reader);
@@ -120,7 +145,7 @@ private Package ReadPackageV13(FileStream mainStream, BinaryReader reader)
             var package = new Package();
             var header = BinUtils.ReadStruct<LSPKHeader13>(reader);
 
-            if (header.Version != (ulong) PackageVersion.V13)
+            if (header.Version != (ulong)PackageVersion.V13)
             {
                 string msg = $"Unsupported package version {header.Version}; this package layout is only supported for {PackageVersion.V13}";
                 throw new InvalidDataException(msg);
@@ -136,7 +161,7 @@ private Package ReadPackageV13(FileStream mainStream, BinaryReader reader)
             mainStream.Seek(header.FileListOffset, SeekOrigin.Begin);
             int numFiles = reader.ReadInt32();
             int fileBufferSize = Marshal.SizeOf(typeof(FileEntry13)) * numFiles;
-            byte[] compressedFileList = reader.ReadBytes((int) header.FileListSize - 4);
+            byte[] compressedFileList = reader.ReadBytes((int)header.FileListSize - 4);
 
             var uncompressedList = new byte[fileBufferSize];
             int uncompressedSize = LZ4Codec.Decode(compressedFileList, 0, compressedFileList.Length, uncompressedList, 0, fileBufferSize, true);
@@ -342,7 +367,7 @@ public Package Read()
                     return ReadPackageV7(mainStream, reader);
                 }
 
-                throw new NotAPackageException("No valid signature found in package file");
+                throw new NotAPackageException("No valid signature found in package file", _path);
             }
         }
     }