Large update to WASM Prism backend with Chicory

.github/workflows/java-wasm-bindings.yml

@@ -38,7 +38,7 @@ jobs:
         uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
-          java-version: '17'
+          java-version: '21'
           cache: maven
 
       - name: Run the tests

.gitignore

@@ -39,9 +39,9 @@ out.svg
 /javascript/src/visitor.js
 /javascript/src/prism.wasm
 /javascript/src/*.d.ts
-/java/org/prism/AbstractNodeVisitor.java
-/java/org/prism/Loader.java
-/java/org/prism/Nodes.java
+/java/org/ruby_lang/prism/AbstractNodeVisitor.java
+/java/org/ruby_lang/prism/Loader.java
+/java/org/ruby_lang/prism/Nodes.java
 /lib/prism/compiler.rb
 /lib/prism/dispatcher.rb
 /lib/prism/dot_visitor.rb
@@ -74,3 +74,5 @@ compile_commands.json
 .vscode/
 
 tags
+
+wasi-sdk*

Gemfile.lock

@@ -65,6 +65,7 @@ GEM
 PLATFORMS
   java
   ruby
+  universal-java
 
 DEPENDENCIES
   benchmark-ips

Makefile

@@ -12,7 +12,7 @@ SOEXT ?= $(shell ruby -e 'puts RbConfig::CONFIG["SOEXT"]')
 
 CPPFLAGS := -Iinclude $(CPPFLAGS)
 CFLAGS := -g -O2 -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -Wno-missing-braces -fPIC -fvisibility=hidden -Wimplicit-fallthrough $(CFLAGS)
-JAVA_WASM_CFLAGS := -g -Oz -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -Wno-missing-braces -fPIC -fvisibility=hidden -Wimplicit-fallthrough $(JAVA_WASM_CFLAGS)
+JAVA_WASM_CFLAGS := -g0 -O2 -Oz -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -Wno-missing-braces -fno-sanitize=all -fno-stack-protector -fPIC -fvisibility=hidden -Wimplicit-fallthrough $(JAVA_WASM_CFLAGS)
 CC ?= cc
 AR ?= ar
 ARFLAGS ?= -r$(V0:1=v)
@@ -53,6 +53,7 @@ javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS)
 
 java-wasm/src/test/resources/prism.wasm: Makefile $(SOURCES) $(HEADERS)
 	$(ECHO) "building $@"
+	$(Q) $(MAKEDIRS) $(@D)
 	$(Q) $(WASI_SDK_PATH)/bin/clang $(DEBUG_FLAGS) -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(JAVA_WASM_CFLAGS) -Wl,--export-all -Wl,--no-entry -mexec-model=reactor -lc++ -lc++abi -o $@ $(SOURCES)
 
 build/shared/%.o: src/%.c Makefile $(HEADERS)

Rakefile

@@ -55,6 +55,7 @@ end
 CLOBBER.concat(Prism::Template::TEMPLATES)
 CLOBBER.concat(["build"])
 CLOBBER << "lib/prism/prism.#{RbConfig::CONFIG["DLEXT"]}"
+CLOBBER << "java-wasm/src/main/resources/prism.wasm"
 
 Prism::Template::TEMPLATES.each do |filepath|
   desc "Generate #{filepath}"

docs/configuration.md

@@ -7,9 +7,9 @@ A lot of code in prism's repository is templated from a single configuration fil
 * `include/prism/diagnostic.h` - for defining the diagnostics
 * `javascript/src/deserialize.js` - for defining how to deserialize the nodes in JavaScript
 * `javascript/src/nodes.js` - for defining the nodes in JavaScript
-* `java/org/prism/AbstractNodeVisitor.java` - for defining the visitor interface for the nodes in Java
-* `java/org/prism/Loader.java` - for defining how to deserialize the nodes in Java
-* `java/org/prism/Nodes.java` - for defining the nodes in Java
+* `java/org/ruby_lang/prism/AbstractNodeVisitor.java` - for defining the visitor interface for the nodes in Java
+* `java/org/ruby_lang/prism/Loader.java` - for defining how to deserialize the nodes in Java
+* `java/org/ruby_lang/prism/Nodes.java` - for defining the nodes in Java
 * `lib/prism/compiler.rb` - for defining the compiler for the nodes in Ruby
 * `lib/prism/dispatcher.rb` - for defining the dispatch visitors for the nodes in Ruby
 * `lib/prism/dot_visitor.rb` - for defining the dot visitor for the nodes in Ruby

docs/releasing.md

@@ -23,9 +23,9 @@ ruby -pi -e 'gsub(/PRISM_VERSION_PATCH \d+/, %Q{PRISM_VERSION_PATCH #{ENV["PRISM
 ruby -pi -e 'gsub(/PRISM_VERSION ".+?"/, %Q{PRISM_VERSION "#{ENV["PRISM_VERSION"]}"})' include/prism/version.h
 ruby -pi -e 'gsub(/"version": ".+?"/, %Q{"version": "#{ENV["PRISM_VERSION"]}"})' javascript/package.json
 ruby -pi -e 'gsub(/lossy\(\), ".+?"/, %Q{lossy(), "#{ENV["PRISM_VERSION"]}"})' rust/ruby-prism-sys/tests/utils_tests.rs
-ruby -pi -e 'gsub(/\d+, "prism major/, %Q{#{ENV["PRISM_MAJOR"]}, "prism major})' templates/java/org/prism/Loader.java.erb
-ruby -pi -e 'gsub(/\d+, "prism minor/, %Q{#{ENV["PRISM_MINOR"]}, "prism minor})' templates/java/org/prism/Loader.java.erb
-ruby -pi -e 'gsub(/\d+, "prism patch/, %Q{#{ENV["PRISM_PATCH"]}, "prism patch})' templates/java/org/prism/Loader.java.erb
+ruby -pi -e 'gsub(/\d+, "prism major/, %Q{#{ENV["PRISM_MAJOR"]}, "prism major})' templates/java/org/ruby_lang/prism/Loader.java.erb
+ruby -pi -e 'gsub(/\d+, "prism minor/, %Q{#{ENV["PRISM_MINOR"]}, "prism minor})' templates/java/org/ruby_lang/prism/Loader.java.erb
+ruby -pi -e 'gsub(/\d+, "prism patch/, %Q{#{ENV["PRISM_PATCH"]}, "prism patch})' templates/java/org/ruby_lang/prism/Loader.java.erb
 ruby -pi -e 'gsub(/MAJOR_VERSION = \d+/, %Q{MAJOR_VERSION = #{ENV["PRISM_MAJOR"]}})' templates/javascript/src/deserialize.js.erb
 ruby -pi -e 'gsub(/MINOR_VERSION = \d+/, %Q{MINOR_VERSION = #{ENV["PRISM_MINOR"]}})' templates/javascript/src/deserialize.js.erb
 ruby -pi -e 'gsub(/PATCH_VERSION = \d+/, %Q{PATCH_VERSION = #{ENV["PRISM_PATCH"]}})' templates/javascript/src/deserialize.js.erb

java-wasm/README.md

@@ -0,0 +1,23 @@
+This dir contains the chicory-prism artifact, a version of prism compiled to WASM and then AOT compiled to JVM bytecode by the Chicory project.
+
+Generate the templated sources:
+
+```
+PRISM_EXCLUDE_PRETTYPRINT=1 PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS=1 PRISM_JAVA_BACKEND=jruby bundle exec rake templates
+```
+
+Compile to WASM using WASI SDK version 25:
+
+```
+make java-wasm WASI_SDK_PATH=.../wasi-sdk-25.0-arm64-macos
+```
+
+Build the AOT-compiled machine and wrapper library:
+
+```
+mvn -f java-wasm/pom.xml clean package
+```
+
+This should build the chicory-wasm jar file and pass some basic tests.
+
+The jar will be under `java-wasm/target/chicory-prism-#####-SNAPSHOT.jar` or can be installed by using `install` instead of `pacakge` in the `mvn` command line above.

java-wasm/perf-test/.gitignore

@@ -0,0 +1 @@
+profile.html

java-wasm/perf-test/bench.sh

@@ -0,0 +1,6 @@
+#! /bin/bash
+set -euxo pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+jbang --fresh --deps org.openjdk.jmh:jmh-generator-annprocess:1.36 --javaagent=ap-loader@jvm-profiling-tools/ap-loader=start,event=cpu,file=${SCRIPT_DIR}/profile.html ${SCRIPT_DIR}/test

java-wasm/perf-test/test

@@ -0,0 +1,97 @@
+///usr/bin/env jbang "$0" "$@" ; exit $?
+
+//DEPS org.jruby:chicory-prism:0.0.1-SNAPSHOT
+//DEPS org.jruby:jruby-complete:10.0.2.0
+
+import static java.lang.System.*;
+
+import java.io.DataInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.EnumSet;
+
+import org.jruby.Ruby;
+
+import org.jruby.parser.prism.wasm.Prism;
+import org.ruby_lang.prism.ParsingOptions;
+
+public class test {
+
+    final static String[] JRUBY_BOOT_FILES = {
+        "jruby/java.rb",
+        "jruby/java/core_ext.rb",
+        "jruby/java/core_ext/object.rb",
+        "jruby/java/java_ext.rb",
+        "jruby/kernel.rb",
+        "jruby/kernel/signal.rb",
+        "jruby/kernel/kernel.rb",
+        "jruby/kernel/proc.rb",
+        "jruby/kernel/process.rb",
+        "jruby/kernel/enumerator.rb",
+        "jruby/kernel/enumerable.rb",
+        "jruby/kernel/io.rb",
+        "jruby/kernel/gc.rb",
+        "jruby/kernel/range.rb",
+        "jruby/kernel/file.rb",
+        "jruby/kernel/method.rb",
+        "jruby/kernel/thread.rb",
+        "jruby/kernel/integer.rb",
+        "jruby/kernel/time.rb",
+        "jruby/preludes.rb",
+        "jruby/kernel/prelude.rb",
+        "jruby/kernel/enc_prelude.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/unicode_normalize.rb",
+        "jruby/kernel/gem_prelude.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rbconfig.rb",
+        "jruby/kernel/rbconfig.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/compatibility.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/defaults.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/deprecate.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/errors.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/target_rbconfig.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/exceptions.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/unknown_command_spell_checker.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/specification.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/basic_specification.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/stub_specification.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/platform.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/specification_record.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/util/list.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/requirement.rb",
+        "META-INF/jruby.home/lib/ruby/stdlib/rubygems/version.rb",
+    };
+
+    public static void main(String... args) throws Exception {
+        out.println("Starting");
+        var count = 100;
+
+        var prism = new Prism();
+        byte[] src = new byte[1024 * 1024];
+
+        for (int i = 0; i < count; i++) {
+            for (var file : JRUBY_BOOT_FILES) {
+                byte[] options = ParsingOptions.serialize(
+                    file.getBytes(StandardCharsets.UTF_8),
+                    1,
+                    "UTF-8".getBytes(StandardCharsets.UTF_8),
+                    false,
+                    EnumSet.noneOf(ParsingOptions.CommandLine.class),
+                    ParsingOptions.SyntaxVersion.LATEST,
+                    false,
+                    false,
+                    false,
+                    new byte[][][]{}
+                );
+
+                try (InputStream fileIn = Ruby.getClassLoader().getResourceAsStream(file)) {
+                    DataInputStream dis = new DataInputStream(fileIn);
+                    int read = dis.read(src);
+                    prism.serialize(options, src, read);
+                }
+            }
+        }
+
+        out.println("End");
+    }
+}

java-wasm/pom.xml

@@ -2,18 +2,18 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
-  <groupId>com.prism</groupId>
-  <artifactId>java-prism</artifactId>
-  <version>999-SNAPSHOT</version>
+  <groupId>org.jruby</groupId>
+  <artifactId>chicory-prism</artifactId>
+  <version>0.0.1-SNAPSHOT</version>
   <name>Java Prism</name>
-  <description>Pure Java Prism interpreting WASM</description>
+  <description>Pure Java Prism using Chicory WASM runtime</description>
 
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-    <maven.compiler.source>11</maven.compiler.source>
-    <maven.compiler.target>11</maven.compiler.target>
+    <maven.compiler.source>21</maven.compiler.source>
+    <maven.compiler.target>21</maven.compiler.target>
 
     <chicory.version>1.6.1</chicory.version>
     <junit.version>6.0.2</junit.version>
@@ -59,6 +59,11 @@
       <artifactId>annotations</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.jruby</groupId>
+      <artifactId>jruby-complete</artifactId>
+      <version>10.0.2.0</version>
+    </dependency>
   </dependencies>
 
   <build>
@@ -81,7 +86,7 @@
         <artifactId>maven-compiler-plugin</artifactId>
         <version>3.14.1</version>
         <configuration>
-          <release>11</release>
+          <release>21</release>
           <annotationProcessorPaths>
             <path>
               <groupId>com.dylibso.chicory</groupId>
@@ -125,7 +130,7 @@
               <goal>compile</goal>
             </goals>
             <configuration>
-              <name>org.prism.PrismModule</name>
+              <name>org.jruby.parser.prism.wasm.PrismParser</name>
               <wasmFile>src/test/resources/prism.wasm</wasmFile>
             </configuration>
           </execution>

java-wasm/src/main/java-templates/org/prism/WasmResource.java → java-wasm/src/main/java-templates/org/jruby/parser/prism/wasm/WasmResource.java

@@ -1,4 +1,4 @@
-package org.prism;
+package org.jruby.parser.prism.wasm;
 
 public final class WasmResource {
     public static final String absoluteFile = "file://${project.basedir}/src/test/resources/prism.wasm";

java-wasm/src/main/java/org/jruby/parser/prism/wasm/Prism.java

@@ -0,0 +1,95 @@
+package org.jruby.parser.prism.wasm;
+
+import com.dylibso.chicory.annotations.WasmModuleInterface;
+import com.dylibso.chicory.runtime.ByteArrayMemory;
+import com.dylibso.chicory.runtime.ImportValues;
+import com.dylibso.chicory.runtime.Instance;
+import com.dylibso.chicory.wasi.WasiOptions;
+import com.dylibso.chicory.wasi.WasiPreview1;
+import org.ruby_lang.prism.Loader;
+import org.ruby_lang.prism.ParseResult;
+
+import java.nio.charset.StandardCharsets;
+
+@WasmModuleInterface(WasmResource.absoluteFile)
+public class Prism implements AutoCloseable {
+    private final WasiPreview1 wasi;
+    protected final Prism_ModuleExports exports;
+    private final Instance instance;
+
+    private int bufferPointer;
+    private int preSourcePointer;
+    private int preOptionsPointer;
+
+    private final int SOURCE_SIZE = 2 * 1024 * 1024; // 2 MiB
+    private final int PACKED_OPTIONS_BUFFER_SIZE = 1024;
+
+    public Prism() {
+        this(WasiOptions.builder().build());
+    }
+
+    public Prism(WasiOptions wasiOpts) {
+        wasi = WasiPreview1.builder().withOptions(wasiOpts).build();
+        instance = Instance.builder(PrismParser.load())
+            .withMemoryFactory(ByteArrayMemory::new)
+            .withMachineFactory(PrismParser::create)
+            .withImportValues(ImportValues.builder().addFunction(wasi.toHostFunctions()).build())
+            .build();
+        exports = new Prism_ModuleExports(instance);
+
+        preOptionsPointer = exports.calloc(1, PACKED_OPTIONS_BUFFER_SIZE);
+        preSourcePointer = exports.calloc(1, SOURCE_SIZE);
+
+        bufferPointer = exports.calloc(exports.pmBufferSizeof(), 1);
+        exports.pmBufferInit(bufferPointer);
+    }
+
+    public byte[] serialize(byte[] packedOptions, byte[] sourceBytes, int sourceLength) {
+        int sourcePointer = 0;
+        boolean useDefaultSourcePointer = sourceLength + 1 > SOURCE_SIZE;
+        int optionsPointer = 0;
+        boolean useDefaultOptionsPointer = packedOptions.length > PACKED_OPTIONS_BUFFER_SIZE;
+        byte[] result;
+        try {
+            sourcePointer = (!useDefaultSourcePointer) ?
+                exports.calloc(1, sourceLength + 1) : preSourcePointer;
+            instance.memory().write(sourcePointer, sourceBytes, 0, sourceLength);
+            instance.memory().writeByte(sourcePointer + sourceLength, (byte) 0);
+
+            optionsPointer = (!useDefaultOptionsPointer) ?
+                exports.calloc(1, packedOptions.length) : preOptionsPointer;
+            instance.memory().write(optionsPointer, packedOptions);
+
+            exports.pmBufferClear(bufferPointer);
+
+            exports.pmSerializeParse(
+                bufferPointer, sourcePointer, sourceLength, optionsPointer);
+
+            result = instance.memory().readBytes(
+                exports.pmBufferValue(bufferPointer),
+                exports.pmBufferLength(bufferPointer));
+        } finally {
+            if (!useDefaultSourcePointer) {
+                exports.free(sourcePointer);
+            }
+            if (!useDefaultOptionsPointer) {
+                exports.free(optionsPointer);
+            }
+        }
+
+        return result;
+    }
+
+    public ParseResult serializeParse(byte[] packedOptions, String source) {
+        var sourceBytes = source.getBytes(StandardCharsets.ISO_8859_1);
+        byte[] result = serialize(packedOptions, sourceBytes, sourceBytes.length);
+        return Loader.load(result, sourceBytes);
+    }
+
+    @Override
+    public void close() {
+        if (wasi != null) {
+            wasi.close();
+        }
+    }
+}