feat: async import loader for JS via static preload (#476)

readme.md

@@ -125,6 +125,30 @@ filesystem, you have to provide an explicit import callback that you can
 use to resolve imports yourself (whether through Node's `fs` module, or
 by emulating a filesystem in-memory)
 
+#### Async imports
+
+`SjsonnetMain.interpret` is synchronous, which is awkward in browsers where
+files come from `fetch` or `FileReader`. Use `SjsonnetMain.interpretAsync`
+instead: the loader returns a `Promise` and the call returns a `Promise` of
+the result. Imports are statically discovered from each parsed file's AST,
+loaded concurrently, then evaluated synchronously against the populated cache.
+
+```javascript
+const result = await SjsonnetMain.interpretAsync(
+  "local lib = import 'lib.libsonnet'; lib.greet('world')",
+  {}, // extVars
+  {}, // tlaVars
+  "", // initial working directory
+  (wd, imported) => imported, // resolver, same shape as `interpret`
+  // loader: returns a Promise of the file contents (string for `import` /
+  // `importstr`, or bytes for `importbin`)
+  async (path, binary) => {
+    const response = await fetch("/files/" + path);
+    return binary ? new Uint8Array(await response.arrayBuffer()) : await response.text();
+  }
+);
+```
+
 ### Running deeply recursive Jsonnet programs
 
 The depth of recursion is limited by running environment stack size. You can run Sjsonnet with increased

sjsonnet/src-js/sjsonnet/SjsonnetMain.scala

@@ -3,7 +3,11 @@ package sjsonnet
 import sjsonnet.stdlib.NativeRegex
 
 import scala.collection.mutable
+import scala.concurrent.Future
+import scala.scalajs.concurrent.JSExecutionContext.Implicits.queue
 import scala.scalajs.js
+import scala.scalajs.js.JSConverters._
+import scala.scalajs.js.Thenable.Implicits._
 import scala.scalajs.js.annotation.{JSExport, JSExportTopLevel}
 import scala.scalajs.js.typedarray.{ArrayBuffer, Int8Array, Uint8Array}
 
@@ -49,6 +53,78 @@ object SjsonnetMain {
     case _ => None
   }
 
+  /** Convert the value returned by a JS import loader into a [[ResolvedFile]]. */
+  private def toResolvedFile(path: String, value: Any, binaryData: Boolean): ResolvedFile =
+    value match {
+      case s: String        => StaticResolvedFile(s)
+      case arr: Array[Byte] => StaticBinaryResolvedFile(arr)
+      case other            =>
+        toBytesFromJs(other) match {
+          case Some(bytes) => StaticBinaryResolvedFile(bytes)
+          case None        =>
+            val msg =
+              s"Import loader for '$path' must return a string or byte array, got: ${
+                  if (other == null) "null" else other.getClass.getName
+                }"
+            js.Dynamic.global.console.error(msg)
+            throw js.JavaScriptException(msg)
+        }
+    }
+
+  /** Build the parent importer used during preload (only its `resolve` is called). */
+  private def jsResolveImporter(importResolver: js.Function2[String, String, String]): Importer =
+    new Importer {
+      def resolve(docBase: Path, importName: String): Option[Path] =
+        importResolver(docBase.asInstanceOf[JsVirtualPath].path, importName) match {
+          case null => None
+          case s    => Some(JsVirtualPath(s))
+        }
+      def read(path: Path, binaryData: Boolean): Option[ResolvedFile] =
+        throw new RuntimeException(
+          s"Importer.read should not be called during async preload (path=$path)"
+        )
+    }
+
+  private def parseStringMap(label: String, value: js.Any): Map[String, String] =
+    try {
+      ujson.WebJson.transform(value, ujson.Value).obj.toMap.map {
+        case (k, ujson.Str(v)) => (k, v)
+        case (k, _)            =>
+          throw js.JavaScriptException(s"$label '$k' must be a string value, got non-string")
+      }
+    } catch {
+      case e: js.JavaScriptException => throw e
+      case e: Exception              =>
+        val msg = s"Failed to parse ${label.toLowerCase}: ${e.getMessage}"
+        js.Dynamic.global.console.error(msg, e.asInstanceOf[js.Any])
+        throw js.JavaScriptException(msg)
+    }
+
+  private def runInterpret(
+      text: String,
+      parsedExtVars: Map[String, String],
+      parsedTlaVars: Map[String, String],
+      wd0: String,
+      importer: Importer,
+      preserveOrder: Boolean): js.Any = {
+    val interp = new Interpreter(
+      parsedExtVars,
+      parsedTlaVars,
+      JsVirtualPath(wd0),
+      importer,
+      parseCache = new DefaultParseCache,
+      settings = new Settings(preserveOrder = preserveOrder),
+      std =
+        new sjsonnet.stdlib.StdLibModule(nativeFunctions = Map.from(NativeRegex.functions)).module
+    )
+    interp.interpret0(text, JsVirtualPath("(memory)"), ujson.WebJson.Builder) match {
+      case Left(msg) =>
+        js.Dynamic.global.console.error("Sjsonnet evaluation error:", msg)
+        throw js.JavaScriptException(msg)
+      case Right(v) => v
+    }
+  }
+
   @JSExport
   def interpret(
       text: String,
@@ -59,85 +135,101 @@ object SjsonnetMain {
       importLoader: js.Function2[String, Boolean, Any],
       preserveOrder: Boolean = false): js.Any = {
     try {
-      val parsedExtVars =
-        try {
-          ujson.WebJson.transform(extVars, ujson.Value).obj.toMap.map {
-            case (k, ujson.Str(v)) => (k, v)
-            case (k, _)            =>
-              throw js.JavaScriptException(
-                s"External variable '$k' must be a string value, got non-string"
-              )
+      val parsedExtVars = parseStringMap("External variable", extVars)
+      val parsedTlaVars = parseStringMap("Top-level argument", tlaVars)
+
+      val importer = new Importer {
+        def resolve(docBase: Path, importName: String): Option[Path] =
+          importResolver(docBase.asInstanceOf[JsVirtualPath].path, importName) match {
+            case null => None
+            case s    => Some(JsVirtualPath(s))
           }
-        } catch {
-          case e: js.JavaScriptException => throw e
-          case e: Exception              =>
-            val msg = s"Failed to parse external variables: ${e.getMessage}"
-            js.Dynamic.global.console.error(msg, e.asInstanceOf[js.Any])
-            throw js.JavaScriptException(msg)
-        }
+        def read(path: Path, binaryData: Boolean): Option[ResolvedFile] =
+          Some(
+            toResolvedFile(
+              path.asInstanceOf[JsVirtualPath].path,
+              importLoader(path.asInstanceOf[JsVirtualPath].path, binaryData),
+              binaryData
+            )
+          )
+      }
 
-      val parsedTlaVars =
-        try {
-          ujson.WebJson.transform(tlaVars, ujson.Value).obj.toMap.map {
-            case (k, ujson.Str(v)) => (k, v)
-            case (k, _)            =>
-              throw js.JavaScriptException(
-                s"Top-level argument '$k' must be a string value, got non-string"
-              )
+      runInterpret(text, parsedExtVars, parsedTlaVars, wd0, importer, preserveOrder)
+    } catch {
+      case e: js.JavaScriptException => throw e
+      case e: Exception              =>
+        val msg = s"Sjsonnet internal error: ${e.getClass.getName}: ${e.getMessage}"
+        js.Dynamic.global.console.error(msg, e.asInstanceOf[js.Any])
+        throw js.JavaScriptException(msg)
+    }
+  }
+
+  /**
+   * Async variant of [[interpret]]. Accepts an `importLoader` that returns a `Promise` of the file
+   * contents, and returns a `Promise` resolving to the rendered output.
+   *
+   * Implementation: imports are statically discovered by parsing each file's AST, loaded
+   * concurrently via the user-supplied async loader, and inserted into a cache. Once the transitive
+   * closure is loaded, evaluation runs synchronously against the cache.
+   */
+  @JSExport
+  def interpretAsync(
+      text: String,
+      extVars: js.Any,
+      tlaVars: js.Any,
+      wd0: String,
+      importResolver: js.Function2[String, String, String],
+      importLoader: js.Function2[String, Boolean, js.Promise[Any]],
+      preserveOrder: Boolean = false): js.Promise[js.Any] = {
+    try {
+      val parsedExtVars = parseStringMap("External variable", extVars)
+      val parsedTlaVars = parseStringMap("Top-level argument", tlaVars)
+
+      val parentImporter = jsResolveImporter(importResolver)
+      val preloader = new Preloader(parentImporter)
+      val entryPath = JsVirtualPath("(memory)")
+
+      preloader.add(entryPath, StaticResolvedFile(text), ImportFinder.Kind.Code) match {
+        case Left(err) => return js.Promise.reject(err.getMessage)
+        case Right(_)  =>
+      }
+
+      def loadOne(p: Preloader.Pending): Future[Unit] = {
+        val pathStr = p.path.asInstanceOf[JsVirtualPath].path
+        val promise = importLoader(pathStr, p.binaryData)
+        // implicit Thenable.Implicits converts Promise[Any] to Future[Any]
+        (promise: Future[Any]).map { value =>
+          val resolved = toResolvedFile(pathStr, value, p.binaryData)
+          preloader.add(p.path, resolved, p.kind) match {
+            case Left(err) => throw js.JavaScriptException(err.getMessage)
+            case Right(_)  => ()
           }
-        } catch {
-          case e: js.JavaScriptException => throw e
-          case e: Exception              =>
-            val msg = s"Failed to parse top-level arguments: ${e.getMessage}"
-            js.Dynamic.global.console.error(msg, e.asInstanceOf[js.Any])
-            throw js.JavaScriptException(msg)
         }
+      }
 
-      val interp = new Interpreter(
-        parsedExtVars,
-        parsedTlaVars,
-        JsVirtualPath(wd0),
-        new Importer {
-          def resolve(docBase: Path, importName: String): Option[Path] =
-            importResolver(docBase.asInstanceOf[JsVirtualPath].path, importName) match {
-              case null => None
-              case s    => Some(JsVirtualPath(s))
-            }
-          def read(path: Path, binaryData: Boolean): Option[ResolvedFile] =
-            importLoader(path.asInstanceOf[JsVirtualPath].path, binaryData) match {
-              case s: String        => Some(StaticResolvedFile(s))
-              case arr: Array[Byte] => Some(StaticBinaryResolvedFile(arr))
-              case other            =>
-                // Handle JS-native binary types: Uint8Array, ArrayBuffer, or plain JS number[]
-                toBytesFromJs(other) match {
-                  case Some(bytes) => Some(StaticBinaryResolvedFile(bytes))
-                  case None        =>
-                    val msg =
-                      s"Import loader for '${path}' must return a string or byte array, got: ${
-                          if (other == null) "null" else other.getClass.getName
-                        }"
-                    js.Dynamic.global.console.error(msg)
-                    throw js.JavaScriptException(msg)
-                }
-            }
-        },
-        parseCache = new DefaultParseCache,
-        settings = new Settings(preserveOrder = preserveOrder),
-        std =
-          new sjsonnet.stdlib.StdLibModule(nativeFunctions = Map.from(NativeRegex.functions)).module
-      )
-      interp.interpret0(text, JsVirtualPath("(memory)"), ujson.WebJson.Builder) match {
-        case Left(msg) =>
-          js.Dynamic.global.console.error("Sjsonnet evaluation error:", msg)
-          throw js.JavaScriptException(msg)
-        case Right(v) => v
+      def loop(): Future[Unit] = {
+        val batch = preloader.takePendingImports()
+        if (batch.isEmpty) Future.successful(())
+        else Future.sequence(batch.map(loadOne)).flatMap(_ => loop())
       }
+
+      val result: Future[js.Any] = loop().map { _ =>
+        runInterpret(
+          text,
+          parsedExtVars,
+          parsedTlaVars,
+          wd0,
+          preloader.importer,
+          preserveOrder
+        )
+      }
+      result.toJSPromise
     } catch {
-      case e: js.JavaScriptException => throw e
+      case e: js.JavaScriptException => js.Promise.reject(e.exception)
       case e: Exception              =>
         val msg = s"Sjsonnet internal error: ${e.getClass.getName}: ${e.getMessage}"
         js.Dynamic.global.console.error(msg, e.asInstanceOf[js.Any])
-        throw js.JavaScriptException(msg)
+        js.Promise.reject(msg)
     }
   }
 }

sjsonnet/src/sjsonnet/ImportFinder.scala

@@ -0,0 +1,57 @@
+package sjsonnet
+
+import scala.collection.mutable
+
+/**
+ * Walks an [[Expr]] AST collecting all `import`, `importstr`, and `importbin` expressions. Used by
+ * [[Preloader]] to discover the transitive set of files that need to be loaded before evaluation.
+ */
+object ImportFinder {
+
+  sealed trait Kind {
+
+    /**
+     * Whether the file should be read as raw bytes (`importbin`) vs. text (`import`/`importstr`).
+     */
+    def binaryData: Boolean
+
+    /** Whether the loaded file is itself Jsonnet code that may contain further imports. */
+    def isCode: Boolean
+  }
+
+  object Kind {
+    case object Code extends Kind {
+      def binaryData: Boolean = false
+      def isCode: Boolean = true
+    }
+    case object Str extends Kind {
+      def binaryData: Boolean = false
+      def isCode: Boolean = false
+    }
+    case object Bin extends Kind {
+      def binaryData: Boolean = true
+      def isCode: Boolean = false
+    }
+  }
+
+  final case class Found(value: String, kind: Kind)
+
+  def collect(expr: Expr): Seq[Found] = {
+    val buf = mutable.ArrayBuffer.empty[Found]
+    val walker = new Walker(buf)
+    walker.transform(expr)
+    buf.toSeq
+  }
+
+  private class Walker(buf: mutable.ArrayBuffer[Found]) extends ExprTransform {
+    override def transform(expr: Expr): Expr = {
+      expr match {
+        case Expr.Import(_, v)    => buf += Found(v, Kind.Code)
+        case Expr.ImportStr(_, v) => buf += Found(v, Kind.Str)
+        case Expr.ImportBin(_, v) => buf += Found(v, Kind.Bin)
+        case _                    =>
+      }
+      rec(expr)
+    }
+  }
+}