From 6758e7a39bee056bb0cd4f2be615e64c277ad3d2 Mon Sep 17 00:00:00 2001 From: Stephen Amar Date: Fri, 1 May 2026 20:39:43 +0000 Subject: [PATCH 1/2] feat: async import loader for JS via static preload (#476) Adds Preloader + ImportFinder to discover imports from each file's AST and drive caller-controlled (async) loading before synchronous evaluation. Exposes SjsonnetMain.interpretAsync for the JS build, accepting a Promise-returning loader so callers can use fetch / FileReader / etc. Co-authored-by: Isaac --- readme.md | 24 ++ sjsonnet/src-js/sjsonnet/SjsonnetMain.scala | 230 ++++++++++++------ sjsonnet/src/sjsonnet/ImportFinder.scala | 57 +++++ sjsonnet/src/sjsonnet/Preloader.scala | 132 ++++++++++ .../src/sjsonnet/stdlib/EncodingModule.scala | 2 - .../src-js/sjsonnet/InterpretAsyncTests.scala | 107 ++++++++ .../test/src/sjsonnet/PreloaderTests.scala | 127 ++++++++++ 7 files changed, 608 insertions(+), 71 deletions(-) create mode 100644 sjsonnet/src/sjsonnet/ImportFinder.scala create mode 100644 sjsonnet/src/sjsonnet/Preloader.scala create mode 100644 sjsonnet/test/src-js/sjsonnet/InterpretAsyncTests.scala create mode 100644 sjsonnet/test/src/sjsonnet/PreloaderTests.scala diff --git a/readme.md b/readme.md index 09fad1a93..3e73e157a 100644 --- a/readme.md +++ b/readme.md @@ -125,6 +125,30 @@ filesystem, you have to provide an explicit import callback that you can use to resolve imports yourself (whether through Node's `fs` module, or by emulating a filesystem in-memory) +#### Async imports + +`SjsonnetMain.interpret` is synchronous, which is awkward in browsers where +files come from `fetch` or `FileReader`. Use `SjsonnetMain.interpretAsync` +instead: the loader returns a `Promise` and the call returns a `Promise` of +the result. Imports are statically discovered from each parsed file's AST, +loaded concurrently, then evaluated synchronously against the populated cache. + +```javascript +const result = await SjsonnetMain.interpretAsync( + "local lib = import 'lib.libsonnet'; lib.greet('world')", + {}, // extVars + {}, // tlaVars + "", // initial working directory + (wd, imported) => imported, // resolver, same shape as `interpret` + // loader: returns a Promise of the file contents (string for `import` / + // `importstr`, or bytes for `importbin`) + async (path, binary) => { + const response = await fetch("/files/" + path); + return binary ? new Uint8Array(await response.arrayBuffer()) : await response.text(); + } +); +``` + ### Running deeply recursive Jsonnet programs The depth of recursion is limited by running environment stack size. You can run Sjsonnet with increased diff --git a/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala b/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala index f96a6cea3..23d7e5c7f 100644 --- a/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala +++ b/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala @@ -3,7 +3,11 @@ package sjsonnet import sjsonnet.stdlib.NativeRegex import scala.collection.mutable +import scala.concurrent.Future +import scala.scalajs.concurrent.JSExecutionContext.Implicits.queue import scala.scalajs.js +import scala.scalajs.js.JSConverters._ +import scala.scalajs.js.Thenable.Implicits._ import scala.scalajs.js.annotation.{JSExport, JSExportTopLevel} import scala.scalajs.js.typedarray.{ArrayBuffer, Int8Array, Uint8Array} @@ -49,6 +53,78 @@ object SjsonnetMain { case _ => None } + /** Convert the value returned by a JS import loader into a [[ResolvedFile]]. */ + private def toResolvedFile(path: String, value: Any, binaryData: Boolean): ResolvedFile = + value match { + case s: String => StaticResolvedFile(s) + case arr: Array[Byte] => StaticBinaryResolvedFile(arr) + case other => + toBytesFromJs(other) match { + case Some(bytes) => StaticBinaryResolvedFile(bytes) + case None => + val msg = + s"Import loader for '$path' must return a string or byte array, got: ${ + if (other == null) "null" else other.getClass.getName + }" + js.Dynamic.global.console.error(msg) + throw js.JavaScriptException(msg) + } + } + + /** Build the parent importer used during preload (only its `resolve` is called). */ + private def jsResolveImporter(importResolver: js.Function2[String, String, String]): Importer = + new Importer { + def resolve(docBase: Path, importName: String): Option[Path] = + importResolver(docBase.asInstanceOf[JsVirtualPath].path, importName) match { + case null => None + case s => Some(JsVirtualPath(s)) + } + def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = + throw new RuntimeException( + s"Importer.read should not be called during async preload (path=$path)" + ) + } + + private def parseStringMap(label: String, value: js.Any): Map[String, String] = + try { + ujson.WebJson.transform(value, ujson.Value).obj.toMap.map { + case (k, ujson.Str(v)) => (k, v) + case (k, _) => + throw js.JavaScriptException(s"$label '$k' must be a string value, got non-string") + } + } catch { + case e: js.JavaScriptException => throw e + case e: Exception => + val msg = s"Failed to parse ${label.toLowerCase}: ${e.getMessage}" + js.Dynamic.global.console.error(msg, e.asInstanceOf[js.Any]) + throw js.JavaScriptException(msg) + } + + private def runInterpret( + text: String, + parsedExtVars: Map[String, String], + parsedTlaVars: Map[String, String], + wd0: String, + importer: Importer, + preserveOrder: Boolean): js.Any = { + val interp = new Interpreter( + parsedExtVars, + parsedTlaVars, + JsVirtualPath(wd0), + importer, + parseCache = new DefaultParseCache, + settings = new Settings(preserveOrder = preserveOrder), + std = + new sjsonnet.stdlib.StdLibModule(nativeFunctions = Map.from(NativeRegex.functions)).module + ) + interp.interpret0(text, JsVirtualPath("(memory)"), ujson.WebJson.Builder) match { + case Left(msg) => + js.Dynamic.global.console.error("Sjsonnet evaluation error:", msg) + throw js.JavaScriptException(msg) + case Right(v) => v + } + } + @JSExport def interpret( text: String, @@ -59,85 +135,101 @@ object SjsonnetMain { importLoader: js.Function2[String, Boolean, Any], preserveOrder: Boolean = false): js.Any = { try { - val parsedExtVars = - try { - ujson.WebJson.transform(extVars, ujson.Value).obj.toMap.map { - case (k, ujson.Str(v)) => (k, v) - case (k, _) => - throw js.JavaScriptException( - s"External variable '$k' must be a string value, got non-string" - ) + val parsedExtVars = parseStringMap("External variable", extVars) + val parsedTlaVars = parseStringMap("Top-level argument", tlaVars) + + val importer = new Importer { + def resolve(docBase: Path, importName: String): Option[Path] = + importResolver(docBase.asInstanceOf[JsVirtualPath].path, importName) match { + case null => None + case s => Some(JsVirtualPath(s)) } - } catch { - case e: js.JavaScriptException => throw e - case e: Exception => - val msg = s"Failed to parse external variables: ${e.getMessage}" - js.Dynamic.global.console.error(msg, e.asInstanceOf[js.Any]) - throw js.JavaScriptException(msg) - } + def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = + Some( + toResolvedFile( + path.asInstanceOf[JsVirtualPath].path, + importLoader(path.asInstanceOf[JsVirtualPath].path, binaryData), + binaryData + ) + ) + } - val parsedTlaVars = - try { - ujson.WebJson.transform(tlaVars, ujson.Value).obj.toMap.map { - case (k, ujson.Str(v)) => (k, v) - case (k, _) => - throw js.JavaScriptException( - s"Top-level argument '$k' must be a string value, got non-string" - ) + runInterpret(text, parsedExtVars, parsedTlaVars, wd0, importer, preserveOrder) + } catch { + case e: js.JavaScriptException => throw e + case e: Exception => + val msg = s"Sjsonnet internal error: ${e.getClass.getName}: ${e.getMessage}" + js.Dynamic.global.console.error(msg, e.asInstanceOf[js.Any]) + throw js.JavaScriptException(msg) + } + } + + /** + * Async variant of [[interpret]]. Accepts an `importLoader` that returns a `Promise` of the file + * contents, and returns a `Promise` resolving to the rendered output. + * + * Implementation: imports are statically discovered by parsing each file's AST, loaded + * concurrently via the user-supplied async loader, and inserted into a cache. Once the transitive + * closure is loaded, evaluation runs synchronously against the cache. + */ + @JSExport + def interpretAsync( + text: String, + extVars: js.Any, + tlaVars: js.Any, + wd0: String, + importResolver: js.Function2[String, String, String], + importLoader: js.Function2[String, Boolean, js.Promise[Any]], + preserveOrder: Boolean = false): js.Promise[js.Any] = { + try { + val parsedExtVars = parseStringMap("External variable", extVars) + val parsedTlaVars = parseStringMap("Top-level argument", tlaVars) + + val parentImporter = jsResolveImporter(importResolver) + val preloader = new Preloader(parentImporter) + val entryPath = JsVirtualPath("(memory)") + + preloader.add(entryPath, StaticResolvedFile(text), ImportFinder.Kind.Code) match { + case Left(err) => return js.Promise.reject(err.getMessage) + case Right(_) => + } + + def loadOne(p: Preloader.Pending): Future[Unit] = { + val pathStr = p.path.asInstanceOf[JsVirtualPath].path + val promise = importLoader(pathStr, p.binaryData) + // implicit Thenable.Implicits converts Promise[Any] to Future[Any] + (promise: Future[Any]).map { value => + val resolved = toResolvedFile(pathStr, value, p.binaryData) + preloader.add(p.path, resolved, p.kind) match { + case Left(err) => throw js.JavaScriptException(err.getMessage) + case Right(_) => () } - } catch { - case e: js.JavaScriptException => throw e - case e: Exception => - val msg = s"Failed to parse top-level arguments: ${e.getMessage}" - js.Dynamic.global.console.error(msg, e.asInstanceOf[js.Any]) - throw js.JavaScriptException(msg) } + } - val interp = new Interpreter( - parsedExtVars, - parsedTlaVars, - JsVirtualPath(wd0), - new Importer { - def resolve(docBase: Path, importName: String): Option[Path] = - importResolver(docBase.asInstanceOf[JsVirtualPath].path, importName) match { - case null => None - case s => Some(JsVirtualPath(s)) - } - def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = - importLoader(path.asInstanceOf[JsVirtualPath].path, binaryData) match { - case s: String => Some(StaticResolvedFile(s)) - case arr: Array[Byte] => Some(StaticBinaryResolvedFile(arr)) - case other => - // Handle JS-native binary types: Uint8Array, ArrayBuffer, or plain JS number[] - toBytesFromJs(other) match { - case Some(bytes) => Some(StaticBinaryResolvedFile(bytes)) - case None => - val msg = - s"Import loader for '${path}' must return a string or byte array, got: ${ - if (other == null) "null" else other.getClass.getName - }" - js.Dynamic.global.console.error(msg) - throw js.JavaScriptException(msg) - } - } - }, - parseCache = new DefaultParseCache, - settings = new Settings(preserveOrder = preserveOrder), - std = - new sjsonnet.stdlib.StdLibModule(nativeFunctions = Map.from(NativeRegex.functions)).module - ) - interp.interpret0(text, JsVirtualPath("(memory)"), ujson.WebJson.Builder) match { - case Left(msg) => - js.Dynamic.global.console.error("Sjsonnet evaluation error:", msg) - throw js.JavaScriptException(msg) - case Right(v) => v + def loop(): Future[Unit] = { + val batch = preloader.takePendingImports() + if (batch.isEmpty) Future.successful(()) + else Future.sequence(batch.map(loadOne)).flatMap(_ => loop()) } + + val result: Future[js.Any] = loop().map { _ => + runInterpret( + text, + parsedExtVars, + parsedTlaVars, + wd0, + preloader.importer, + preserveOrder + ) + } + result.toJSPromise } catch { - case e: js.JavaScriptException => throw e + case e: js.JavaScriptException => js.Promise.reject(e.exception) case e: Exception => val msg = s"Sjsonnet internal error: ${e.getClass.getName}: ${e.getMessage}" js.Dynamic.global.console.error(msg, e.asInstanceOf[js.Any]) - throw js.JavaScriptException(msg) + js.Promise.reject(msg) } } } diff --git a/sjsonnet/src/sjsonnet/ImportFinder.scala b/sjsonnet/src/sjsonnet/ImportFinder.scala new file mode 100644 index 000000000..501ad9489 --- /dev/null +++ b/sjsonnet/src/sjsonnet/ImportFinder.scala @@ -0,0 +1,57 @@ +package sjsonnet + +import scala.collection.mutable + +/** + * Walks an [[Expr]] AST collecting all `import`, `importstr`, and `importbin` expressions. Used by + * [[Preloader]] to discover the transitive set of files that need to be loaded before evaluation. + */ +object ImportFinder { + + sealed trait Kind { + + /** + * Whether the file should be read as raw bytes (`importbin`) vs. text (`import`/`importstr`). + */ + def binaryData: Boolean + + /** Whether the loaded file is itself Jsonnet code that may contain further imports. */ + def isCode: Boolean + } + + object Kind { + case object Code extends Kind { + def binaryData: Boolean = false + def isCode: Boolean = true + } + case object Str extends Kind { + def binaryData: Boolean = false + def isCode: Boolean = false + } + case object Bin extends Kind { + def binaryData: Boolean = true + def isCode: Boolean = false + } + } + + final case class Found(value: String, kind: Kind) + + def collect(expr: Expr): Seq[Found] = { + val buf = mutable.ArrayBuffer.empty[Found] + val walker = new Walker(buf) + walker.transform(expr) + buf.toSeq + } + + private class Walker(buf: mutable.ArrayBuffer[Found]) extends ExprTransform { + override def transform(expr: Expr): Expr = { + expr match { + case Expr.Import(_, v) => buf += Found(v, Kind.Code) + case Expr.ImportStr(_, v) => buf += Found(v, Kind.Str) + case Expr.ImportBin(_, v) => buf += Found(v, Kind.Bin) + case _ => + } + rec(expr) + } + } +} diff --git a/sjsonnet/src/sjsonnet/Preloader.scala b/sjsonnet/src/sjsonnet/Preloader.scala new file mode 100644 index 000000000..b1d72ab8e --- /dev/null +++ b/sjsonnet/src/sjsonnet/Preloader.scala @@ -0,0 +1,132 @@ +package sjsonnet + +import fastparse.Parsed + +import scala.collection.mutable + +/** + * Drives asynchronous (or otherwise externally-controlled) loading of imports by statically + * discovering them ahead of evaluation. + * + * Jsonnet has no dynamic imports: every `import`, `importstr`, or `importbin` expression has a + * literal string path. So given the parsed AST of an entry file we can enumerate its imports, load + * them, parse the loaded code files for further imports, and repeat until the closure is known. + * Once all files are in the cache, normal synchronous evaluation can run. + * + * Usage (pseudo-code): + * {{{ + * val preloader = new Preloader(parentImporter) + * preloader.add(entryPath, StaticResolvedFile(entryText), Preloader.EntryKind) + * while (preloader.pendingImports.nonEmpty) { + * val batch = preloader.takePendingImports() + * for (p <- batch) { + * val content = await asyncLoad(p.path, p.binaryData) // platform-specific async + * preloader.add(p.path, content, p.kind) + * } + * } + * val interpreter = new Interpreter(..., importer = preloader.importer, ...) + * interpreter.interpret(entryText, entryPath) + * }}} + * + * @param parentImporter + * used only to resolve import names to [[Path]]s. Its `read` is never called. + * @param settings + * parser settings (recursion depth, etc.). + */ +class Preloader(parentImporter: Importer, settings: Settings = Settings.default) { + + private val internedStrings = new mutable.HashMap[String, String] + private val internedFieldSets = + new mutable.HashMap[Val.StaticObjectFieldSet, java.util.LinkedHashMap[ + String, + java.lang.Boolean + ]] + + private val cache = mutable.LinkedHashMap.empty[Path, ResolvedFile] + private val seen = mutable.HashSet.empty[(Path, ImportFinder.Kind)] + private val pending = mutable.ArrayBuffer.empty[Preloader.Pending] + + /** Resolve an import name relative to a base path, using the parent importer. */ + def resolve(docBase: Path, importName: String): Option[Path] = + parentImporter.resolve(docBase, importName) + + /** + * Register a loaded file in the cache and, if it's a Jsonnet code file, parse it to discover its + * imports. + * + * The returned `Either` reports parse errors for code files; binary or string imports never fail + * here. + */ + def add( + path: Path, + content: ResolvedFile, + kind: ImportFinder.Kind = ImportFinder.Kind.Code): Either[Error, Unit] = { + cache.put(path, content) + if (kind.isCode) discover(path, content) else Right(()) + } + + /** All imports queued for loading. */ + def pendingImports: Seq[Preloader.Pending] = pending.toSeq + + /** Atomically take and clear the queue of pending imports. */ + def takePendingImports(): Seq[Preloader.Pending] = { + val out = pending.toVector + pending.clear() + out + } + + /** True when no more imports need to be loaded. */ + def isComplete: Boolean = pending.isEmpty + + /** + * An [[Importer]] that resolves names through the parent importer but reads exclusively from this + * preloader's cache. Pass to an [[Interpreter]] for synchronous evaluation after preload + * completes. + */ + def importer: Importer = new Importer { + def resolve(docBase: Path, importName: String): Option[Path] = + parentImporter.resolve(docBase, importName) + def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = cache.get(path) + } + + /** Snapshot of the loaded cache, exposed so callers can inspect or persist it. */ + def loaded: collection.Map[Path, ResolvedFile] = cache + + private def discover(path: Path, content: ResolvedFile): Either[Error, Unit] = { + val parser = new Parser(path, internedStrings, internedFieldSets, settings) + try { + fastparse.parse(content.getParserInput(), parser.document(_)) match { + case f: Parsed.Failure => + val traced = f.trace() + Left(new ParseError(s"$path: ${traced.msg}", offset = traced.index)) + case Parsed.Success((expr, _), _) => + ImportFinder.collect(expr).foreach { found => + parentImporter.resolve(path, found.value) match { + case Some(resolved) => + if (seen.add((resolved, found.kind))) enqueue(resolved, found.kind) + case None => + // resolution failure is deferred until evaluation, where it will surface + // with a proper stack frame + } + } + Right(()) + } + } catch { + case e: ParseError => Left(e) + } + } + + private def enqueue(path: Path, kind: ImportFinder.Kind): Unit = { + if (cache.contains(path)) return + pending += Preloader.Pending(path, kind) + } +} + +object Preloader { + + /** A path that needs to be loaded before evaluation can proceed. */ + final case class Pending(path: Path, kind: ImportFinder.Kind) { + def binaryData: Boolean = kind.binaryData + def isCode: Boolean = kind.isCode + } +} diff --git a/sjsonnet/src/sjsonnet/stdlib/EncodingModule.scala b/sjsonnet/src/sjsonnet/stdlib/EncodingModule.scala index f2375440c..f65456487 100644 --- a/sjsonnet/src/sjsonnet/stdlib/EncodingModule.scala +++ b/sjsonnet/src/sjsonnet/stdlib/EncodingModule.scala @@ -5,8 +5,6 @@ import java.nio.charset.StandardCharsets.UTF_8 import sjsonnet._ import sjsonnet.functions.AbstractFunctionModule -import java.nio.charset.StandardCharsets.UTF_8 - /** * Native implementations for Jsonnet standard-library entries in this module. * diff --git a/sjsonnet/test/src-js/sjsonnet/InterpretAsyncTests.scala b/sjsonnet/test/src-js/sjsonnet/InterpretAsyncTests.scala new file mode 100644 index 000000000..187cdd91c --- /dev/null +++ b/sjsonnet/test/src-js/sjsonnet/InterpretAsyncTests.scala @@ -0,0 +1,107 @@ +package sjsonnet + +import utest._ + +import scala.concurrent.Future +import scala.scalajs.concurrent.JSExecutionContext.Implicits.queue +import scala.scalajs.js +import scala.scalajs.js.JSConverters._ + +object InterpretAsyncTests extends TestSuite { + + /** + * Wraps a synchronous file map as a JS Promise-returning loader, so the test exercises the real + * async code path. Records the order of resolved loads. + */ + private def makeAsyncLoader( + files: Map[String, String], + bin: Map[String, Array[Byte]] = Map.empty, + log: scala.collection.mutable.ArrayBuffer[String] = + scala.collection.mutable.ArrayBuffer.empty) + : js.Function2[String, Boolean, js.Promise[Any]] = { + (path: String, binaryData: Boolean) => + Future { + log += path + if (binaryData) { + bin.get(path) match { + case Some(b) => b.toJSArray.asInstanceOf[Any] + case None => throw js.JavaScriptException(s"missing binary file: $path") + } + } else { + files.get(path) match { + case Some(s) => s.asInstanceOf[Any] + case None => throw js.JavaScriptException(s"missing file: $path") + } + } + }.toJSPromise + } + + private def makeResolver(known: Set[String]): js.Function2[String, String, String] = + (_: String, name: String) => if (known.contains(name)) name else null + + private def runAsync( + text: String, + files: Map[String, String] = Map.empty, + bin: Map[String, Array[Byte]] = Map.empty): Future[ujson.Value] = { + val loader = makeAsyncLoader(files, bin) + val resolver = makeResolver(files.keySet ++ bin.keySet) + SjsonnetMain + .interpretAsync( + text, + js.Dictionary[js.Any](), + js.Dictionary[js.Any](), + "/", + resolver, + loader + ) + .toFuture + .map(v => ujson.WebJson.transform(v, ujson.Value)) + } + + def tests: Tests = Tests { + + test("simple async import returns a Promise of the result") { + runAsync( + "(import 'lib.libsonnet').n", + Map("lib.libsonnet" -> "{ n: 42 }") + ).map(v => assert(v == ujson.Num(42))) + } + + test("transitive async imports load and evaluate") { + runAsync( + "(import 'a.libsonnet').value", + Map( + "a.libsonnet" -> "{ value: (import 'b.libsonnet').y + 1 }", + "b.libsonnet" -> "{ y: 10 }" + ) + ).map(v => assert(v == ujson.Num(11))) + } + + test("importstr loads as text without further parsing") { + // The data file would be invalid Jsonnet — it must NOT be parsed. + runAsync( + "importstr 'data.txt'", + Map("data.txt" -> "this is :: not :: jsonnet") + ).map(v => assert(v == ujson.Str("this is :: not :: jsonnet"))) + } + + test("async loader rejection propagates through the returned Promise") { + val resolver = makeResolver(Set("missing.libsonnet")) + val loader: js.Function2[String, Boolean, js.Promise[Any]] = + (path: String, _: Boolean) => js.Promise.reject(s"boom: $path") + val out = SjsonnetMain.interpretAsync( + "import 'missing.libsonnet'", + js.Dictionary[js.Any](), + js.Dictionary[js.Any](), + "/", + resolver, + loader + ) + out.toFuture.transform { + case scala.util.Failure(_) => scala.util.Success(()) + case scala.util.Success(v) => + scala.util.Failure(new RuntimeException(s"expected failure, got $v")) + } + } + } +} diff --git a/sjsonnet/test/src/sjsonnet/PreloaderTests.scala b/sjsonnet/test/src/sjsonnet/PreloaderTests.scala new file mode 100644 index 000000000..0fe694647 --- /dev/null +++ b/sjsonnet/test/src/sjsonnet/PreloaderTests.scala @@ -0,0 +1,127 @@ +package sjsonnet + +import utest._ + +import scala.collection.mutable + +object PreloaderTests extends TestSuite { + + /** A virtual file system used by both the preloader's `resolve` and the test's loading loop. */ + private class FakeFs(files: Map[String, String], binFiles: Map[String, Array[Byte]] = Map.empty) { + val readPaths: mutable.ArrayBuffer[(String, Boolean)] = mutable.ArrayBuffer.empty + + val importer: Importer = new Importer { + def resolve(docBase: Path, importName: String): Option[Path] = { + val candidate = DummyPath(importName) + if (files.contains(importName) || binFiles.contains(importName)) Some(candidate) else None + } + def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = + throw new RuntimeException(s"read should not be called during preload: $path") + } + + def load(path: Path, binaryData: Boolean): ResolvedFile = { + val key = path.asInstanceOf[DummyPath].segments.head + readPaths += ((key, binaryData)) + if (binaryData) StaticBinaryResolvedFile(binFiles(key)) + else StaticResolvedFile(files(key)) + } + } + + private def runPreload(fs: FakeFs, entryPath: Path, entry: String): Preloader = { + val preloader = new Preloader(fs.importer) + preloader.add(entryPath, StaticResolvedFile(entry), ImportFinder.Kind.Code) match { + case Left(err) => throw err + case Right(_) => + } + while (!preloader.isComplete) { + val batch = preloader.takePendingImports() + batch.foreach { p => + val content = fs.load(p.path, p.binaryData) + preloader.add(p.path, content, p.kind) match { + case Left(err) => throw err + case Right(_) => + } + } + } + preloader + } + + def tests: Tests = Tests { + + test("discovers transitive imports") { + val fs = new FakeFs( + Map( + "a.libsonnet" -> "import 'b.libsonnet'", + "b.libsonnet" -> "{ x: 1 }" + ) + ) + val entry = "import 'a.libsonnet'" + val preloader = runPreload(fs, DummyPath("entry"), entry) + + val loaded = fs.readPaths.map(_._1).toSet + assert(loaded == Set("a.libsonnet", "b.libsonnet")) + assert(preloader.loaded.size == 3) // entry + a + b + } + + test("dedupes identical imports") { + val fs = new FakeFs( + Map( + "shared.libsonnet" -> "{ y: 2 }" + ) + ) + val entry = "[import 'shared.libsonnet', import 'shared.libsonnet']" + runPreload(fs, DummyPath("entry"), entry) + + assert(fs.readPaths.count(_._1 == "shared.libsonnet") == 1) + } + + test("handles importstr and importbin") { + val fs = new FakeFs( + Map("data.txt" -> "hello"), + binFiles = Map("blob.bin" -> Array[Byte](1, 2, 3)) + ) + val entry = "{ s: importstr 'data.txt', b: importbin 'blob.bin' }" + runPreload(fs, DummyPath("entry"), entry) + + assert(fs.readPaths.toSet == Set(("data.txt", false), ("blob.bin", true))) + } + + test("does not parse importstr/importbin contents for further imports") { + // The string content here would be invalid Jsonnet if parsed; preloader must not parse it. + val fs = new FakeFs(Map("data.txt" -> "this is not jsonnet ::: !@#")) + val entry = "importstr 'data.txt'" + runPreload(fs, DummyPath("entry"), entry) + + assert(fs.readPaths.toSeq == Seq(("data.txt", false))) + } + + test("interpreter evaluates against preloaded cache") { + val fs = new FakeFs( + Map( + "lib.libsonnet" -> "{ greet(name): 'hello, ' + name }" + ) + ) + val entry = "(import 'lib.libsonnet').greet('world')" + val entryPath = DummyPath("entry") + val preloader = runPreload(fs, entryPath, entry) + + val interp = new Interpreter( + Map.empty[String, String], + Map.empty[String, String], + DummyPath(), + preloader.importer, + parseCache = new DefaultParseCache + ) + val result = interp.interpret(entry, entryPath) + assert(result == Right(ujson.Str("hello, world"))) + } + + test("parse error in entry is reported") { + val fs = new FakeFs(Map.empty) + val preloader = new Preloader(fs.importer) + val out = + preloader.add(DummyPath("entry"), StaticResolvedFile("local x ="), ImportFinder.Kind.Code) + assert(out.isLeft) + } + } +} From cc2848f98f3934cc388ef2567269f5b3d137bacb Mon Sep 17 00:00:00 2001 From: Stephen Amar Date: Fri, 1 May 2026 23:40:09 +0000 Subject: [PATCH 2/2] test: drop unused defaults in InterpretAsyncTests for Scala 2.13/2.12 Scala 2.13/2.12 promote "private default argument never used" to an error; the binary-loader and log defaults in the test helpers were unused at every call site. Inline them since no test exercises binary imports through the async path. Co-authored-by: Isaac --- .../src-js/sjsonnet/InterpretAsyncTests.scala | 33 +++++-------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/sjsonnet/test/src-js/sjsonnet/InterpretAsyncTests.scala b/sjsonnet/test/src-js/sjsonnet/InterpretAsyncTests.scala index 187cdd91c..a3cc8346b 100644 --- a/sjsonnet/test/src-js/sjsonnet/InterpretAsyncTests.scala +++ b/sjsonnet/test/src-js/sjsonnet/InterpretAsyncTests.scala @@ -11,27 +11,15 @@ object InterpretAsyncTests extends TestSuite { /** * Wraps a synchronous file map as a JS Promise-returning loader, so the test exercises the real - * async code path. Records the order of resolved loads. + * async code path. */ private def makeAsyncLoader( - files: Map[String, String], - bin: Map[String, Array[Byte]] = Map.empty, - log: scala.collection.mutable.ArrayBuffer[String] = - scala.collection.mutable.ArrayBuffer.empty) - : js.Function2[String, Boolean, js.Promise[Any]] = { - (path: String, binaryData: Boolean) => + files: Map[String, String]): js.Function2[String, Boolean, js.Promise[Any]] = { + (path: String, _: Boolean) => Future { - log += path - if (binaryData) { - bin.get(path) match { - case Some(b) => b.toJSArray.asInstanceOf[Any] - case None => throw js.JavaScriptException(s"missing binary file: $path") - } - } else { - files.get(path) match { - case Some(s) => s.asInstanceOf[Any] - case None => throw js.JavaScriptException(s"missing file: $path") - } + files.get(path) match { + case Some(s) => s.asInstanceOf[Any] + case None => throw js.JavaScriptException(s"missing file: $path") } }.toJSPromise } @@ -39,12 +27,9 @@ object InterpretAsyncTests extends TestSuite { private def makeResolver(known: Set[String]): js.Function2[String, String, String] = (_: String, name: String) => if (known.contains(name)) name else null - private def runAsync( - text: String, - files: Map[String, String] = Map.empty, - bin: Map[String, Array[Byte]] = Map.empty): Future[ujson.Value] = { - val loader = makeAsyncLoader(files, bin) - val resolver = makeResolver(files.keySet ++ bin.keySet) + private def runAsync(text: String, files: Map[String, String]): Future[ujson.Value] = { + val loader = makeAsyncLoader(files) + val resolver = makeResolver(files.keySet) SjsonnetMain .interpretAsync( text,