diff --git a/rustler/src/nif.rs b/rustler/src/nif.rs index cd0122276..db77a8c6c 100644 --- a/rustler/src/nif.rs +++ b/rustler/src/nif.rs @@ -2,6 +2,7 @@ use crate::sys::{c_char, c_int, c_uint, ErlNifEnv, ErlNifFunc, ErlNifTerm}; pub struct Nif { pub name: *const c_char, + pub signature: &'static str, pub arity: c_uint, pub flags: c_uint, // pub func: DEF_NIF_FUNC, diff --git a/rustler_codegen/src/nif.rs b/rustler_codegen/src/nif.rs index 3f8f88dbc..449877d13 100644 --- a/rustler_codegen/src/nif.rs +++ b/rustler_codegen/src/nif.rs @@ -5,6 +5,7 @@ use syn::punctuated::Punctuated; use syn::token::Comma; use syn::LitStr; +const RUSTLER_TAG: &str = "?RUSTLER-NIF-SIG-V1"; const VALID_SCHEDULE_OPTIONS: [&str; 3] = ["Normal", "DirtyCpu", "DirtyIo"]; #[derive(Default)] @@ -53,11 +54,14 @@ pub fn transcoder_decorator(nif_attributes: NifAttributes, fun: syn::ItemFn) -> panic!("Only non-Control ASCII strings are supported as function names"); } + let signature_s = format!("<{RUSTLER_TAG}>{erl_func_name}/{arity}"); + quote! { rustler::codegen_runtime::inventory::submit!( rustler::Nif { name: concat!(#erl_func_name, "\0").as_ptr() as *const rustler::codegen_runtime::c_char, + signature: #signature_s, arity: #arity, flags: #flags as rustler::codegen_runtime::c_uint, raw_func: { diff --git a/rustler_mix/lib/rustler.ex b/rustler_mix/lib/rustler.ex index feab6dbde..f978bb06f 100644 --- a/rustler_mix/lib/rustler.ex +++ b/rustler_mix/lib/rustler.ex @@ -93,30 +93,72 @@ defmodule Rustler do defmacro __using__(opts) do quote bind_quoted: [opts: opts] do + import Rustler, only: [sigil_CARGO: 2, sigil_RUST: 2] + otp_app = Keyword.fetch!(opts, :otp_app) env = Application.compile_env(otp_app, __MODULE__, []) - config = Rustler.Compiler.compile_crate(otp_app, env, opts) - for resource <- config.external_resources do - @external_resource resource - end + Module.register_attribute(__MODULE__, :rustler_cargo, accumulate: true) + Module.register_attribute(__MODULE__, :rustler_rust, accumulate: true) + @rustler_opts opts + @rustler_config env + @before_compile Rustler + end + end - if config.lib do - @load_from config.load_from - @load_data config.load_data - @load_data_fun config.load_data_fun + defmacro __before_compile__(env) do + opts = Module.get_attribute(env.module, :rustler_opts) + config_env = Module.get_attribute(env.module, :rustler_config) + cargo_blocks = Module.get_attribute(env.module, :rustler_cargo) |> Enum.reverse() + rust_blocks = Module.get_attribute(env.module, :rustler_rust) |> Enum.reverse() + otp_app = Keyword.fetch!(opts, :otp_app) + crate = config_env[:crate] || opts[:crate] || otp_app + + opts = + if cargo_blocks == [] and rust_blocks == [] do + opts + else + path = + config_env[:path] || opts[:path] || + if mix_application?() do + Path.join(["native", to_string(crate)]) + else + Path.join([".rustler", to_string(crate)]) + end + + generate_crate(path, cargo_blocks, rust_blocks) + Keyword.put(opts, :path, path) + end - @before_compile Rustler + config = Rustler.Compiler.compile_crate(otp_app, config_env, opts) + + resources = + Enum.map(config.external_resources, fn resource -> + quote do + @external_resource unquote(resource) + end + end) + |> Kernel.++([ + quote do + @external_resource unquote(Rustler.Compiler.signature_file(config)) + end + ]) + + if config.lib do + compile_module(env, config, resources) + else + quote do + (unquote_splicing(resources)) end end end - defmacro __before_compile__(env) do + defp compile_module(env, config, resources) do default_load_data_value = %Rustler.Compiler.Config{}.load_data default_fun_value = %Rustler.Compiler.Config{}.load_data_fun - load_data = Module.get_attribute(env.module, :load_data) - load_data_fun = Module.get_attribute(env.module, :load_data_fun) + load_data = config.load_data + load_data_fun = config.load_data_fun load_data = case {load_data, load_data_fun} do @@ -145,7 +187,22 @@ defmodule Rustler do """ end + stubs = + for {name, arity} <- config.signatures, + fun = String.to_atom(name), + not Module.defines?(env.module, {fun, arity}) do + args = Macro.generate_arguments(arity, env.module) + + quote do + @doc false + def unquote(fun)(unquote_splicing(args)), do: :erlang.nif_error(:nif_not_loaded) + end + end + quote do + unquote_splicing(resources) + unquote_splicing(stubs) + @on_load :rustler_init @doc false @@ -154,7 +211,7 @@ defmodule Rustler do # {:error, {:upgrade, 'Upgrade not supported by this NIF library.'}} :code.purge(__MODULE__) - {otp_app, path} = @load_from + {otp_app, path} = unquote(Macro.escape(config.load_from)) load_path = otp_app @@ -167,4 +224,34 @@ defmodule Rustler do end end end + + defp mix_application? do + Process.whereis(Mix.ProjectStack) && Mix.Project.config()[:app] + end + + defp generate_crate(path, cargo_blocks, rust_blocks) do + if rust_blocks != [] do + if cargo_blocks == [] do + Mix.shell().error("Must define a ~CARGO block for inline ~RUST") + raise "Missing ~CARGO block" + end + + File.mkdir_p!(Path.join(path, "src")) + + File.write!(Path.join(path, "Cargo.toml"), Enum.join(cargo_blocks, "\n")) + File.write!(Path.join([path, "src", "lib.rs"]), Enum.join(rust_blocks, "\n")) + end + end + + defmacro sigil_CARGO({:<<>>, _, _} = content, []) do + quote do + @rustler_cargo unquote(content) + end + end + + defmacro sigil_RUST({:<<>>, _, _} = content, []) do + quote do + @rustler_rust unquote(content) + end + end end diff --git a/rustler_mix/lib/rustler/compiler.ex b/rustler_mix/lib/rustler/compiler.ex index 8b3f18c6c..7239c86c4 100644 --- a/rustler_mix/lib/rustler/compiler.ex +++ b/rustler_mix/lib/rustler/compiler.ex @@ -32,7 +32,31 @@ defmodule Rustler.Compiler do end end - config + %{config | signatures: read_signatures(config)} + end + + @doc false + def signature_file(config) do + if config.skip_compilation? do + {otp_app, path} = config.load_from + :rustler.signature_path(Application.app_dir(otp_app, path)) + else + :rustler.signature_path( + Path.join(priv_dir(), output_file(config.crate, :lib, config.target)) + ) + end + end + + defp read_signatures(config) do + if config.lib do + case :rustler.read_signatures(signature_file(config)) do + {:ok, signatures} -> signatures + {:error, :enoent} -> [] + {:error, reason} -> raise "Failed to read Rustler signatures: #{inspect(reason)}" + end + else + [] + end end defp do_compile(config, recompile \\ false) do @@ -155,6 +179,19 @@ defmodule Rustler.Compiler do Mix.shell().info("Copying #{artifact.filename} to #{destination_lib}") File.rm(destination_lib) File.cp!(artifact.filename, destination_lib) + + if artifact.kind == :lib do + case :rustler.write_signatures( + destination_lib, + :rustler.signature_path(destination_lib) + ) do + :ok -> + :ok + + {:error, reason} -> + raise "Failed to write Rustler signatures: #{inspect(reason)}" + end + end end end) end diff --git a/rustler_mix/lib/rustler/compiler/config.ex b/rustler_mix/lib/rustler/compiler/config.ex index c71226573..7867e0e01 100644 --- a/rustler_mix/lib/rustler/compiler/config.ex +++ b/rustler_mix/lib/rustler/compiler/config.ex @@ -25,6 +25,7 @@ defmodule Rustler.Compiler.Config do mode: :release, otp_app: nil, path: "", + signatures: [], skip_compilation?: false, target: nil, metadata: nil diff --git a/rustler_mix/src/rustler.erl b/rustler_mix/src/rustler.erl new file mode 100644 index 000000000..684c2fd33 --- /dev/null +++ b/rustler_mix/src/rustler.erl @@ -0,0 +1,127 @@ +-module(rustler). + +-export([ + get_path/2, + get_signatures/2, + signature_path/1, + write_signatures/2, + read_signatures/1 +]). + +-define(CHUNK_SIZE, 65536). % 64 KB chunks + +get_path(Application, Crate) -> + filename:join([ + code:priv_dir(Application), + binary:join([Crate, platform_ext()], <<".">>) + ]). + +get_signatures(Application, Crate) -> + Path = get_path(Application, Crate), + extract_signatures(Path, 1). + +signature_path(LibraryPath) -> + RootPath = filename:rootname(LibraryPath, filename:extension(LibraryPath)), + <>. + +write_signatures(LibraryPath, SignaturePath) -> + case extract_signatures(LibraryPath, 1) of + Signatures when is_list(Signatures) -> + Lines = [signature_line(Signature) || Signature <- Signatures], + file:write_file(SignaturePath, Lines); + {error, Reason} -> + {error, Reason} + end. + +read_signatures(SignaturePath) -> + case file:read_file(SignaturePath) of + {ok, Content} -> + {ok, [parse_signature_line(Line) || + Line <- binary:split(Content, <<"\n">>, [global]), + Line =/= <<>>]}; + {error, Reason} -> + {error, Reason} + end. + +signature_line(Signature) -> + {Name, Arity} = parse_signature_line(Signature), + [Name, "/", integer_to_binary(Arity), "\n"]. + +parse_signature_line(Signature) -> + [ArityBin | ReversedName] = + lists:reverse(binary:split(Signature, <<"/">>, [global])), + {iolist_to_binary(lists:join(<<"/">>, lists:reverse(ReversedName))), + binary_to_integer(ArityBin)}. + +platform_ext() -> + platform_ext(os:type()). + +platform_ext(Platform) -> + case Platform of + {unix, _} -> <<".so">>; + {win32, _} -> <<".dll">> + end. + +%% Public API: Pass the target version (e.g., 1) +extract_signatures(FilePath, Version) when is_integer(Version) -> + VBin = integer_to_binary(Version), + StartTag = iolist_to_binary([<<">, VBin, <<">">>]), + EndTag = iolist_to_binary([<<">, VBin, <<">">>]), + + case file:open(FilePath, [read, binary, raw, {read_ahead, ?CHUNK_SIZE}]) of + {ok, IoDevice} -> + try + read_chunks(IoDevice, StartTag, EndTag, <<>>, []) + after + file:close(IoDevice) + end; + {error, Reason} -> + {error, Reason} + end. + +%% Internal: Stream chunks through file:read/2 +read_chunks(IoDevice, StartTag, EndTag, AccBin, Signatures) -> + case file:read(IoDevice, ?CHUNK_SIZE) of + {ok, Data} -> + {NewAcc, NewSigs} = parse_binary(<>, StartTag, EndTag, Signatures), + read_chunks(IoDevice, StartTag, EndTag, NewAcc, NewSigs); + eof -> + {_, FinalSigs} = parse_binary(AccBin, StartTag, EndTag, Signatures), + lists:reverse(FinalSigs); + {error, Reason} -> + {error, Reason} + end. + +%% Internal: Dynamic matching with memory cleanup & boundary protection +parse_binary(Bin, StartTag, EndTag, Signatures) -> + case binary:match(Bin, StartTag) of + {StartPos, StartLen} -> + <<_:StartPos/binary, _:StartLen/binary, Rest/binary>> = Bin, + case binary:match(Rest, EndTag) of + {EndPos, EndTagLen} -> + <> = Rest, + <<_:EndPos/binary, _:EndTagLen/binary, Remaining/binary>> = Rest, + + % Copy signature to local heap, releasing the massive chunk buffer + CopiedSig = binary:copy(Sig), + parse_binary(Remaining, StartTag, EndTag, [CopiedSig | Signatures]); + nomatch -> + % End tag is cut off by chunk boundary. Keep from StartTag onwards. + <<_:StartPos/binary, KeepBin/binary>> = Bin, + {KeepBin, Signatures} + end; + nomatch -> + % No start tag found. Keep only the trailing tail (TagLen - 1 bytes) + % in case a start tag was split right across the chunk boundary. + TagLen = byte_size(StartTag), + BinLen = byte_size(Bin), + if + BinLen >= TagLen -> + KeepLen = TagLen - 1, + SkipLen = BinLen - KeepLen, + <<_:SkipLen/binary, KeepBin/binary>> = Bin, + {KeepBin, Signatures}; + true -> + {Bin, Signatures} + end + end. diff --git a/rustler_mix/test/sigil_test.exs b/rustler_mix/test/sigil_test.exs new file mode 100644 index 000000000..c273d7622 --- /dev/null +++ b/rustler_mix/test/sigil_test.exs @@ -0,0 +1,51 @@ +defmodule Rustler.SigilTest do + use ExUnit.Case, async: false + + test "writes collected CARGO and RUST sigils into a crate" do + crate_path = + Path.join( + System.tmp_dir!(), + "rustler-sigil-#{System.unique_integer([:positive])}" + ) + + on_exit(fn -> File.rm_rf!(crate_path) end) + + module = "Rustler.SigilGenerated#{System.unique_integer([:positive])}" + + Code.compile_string(""" + defmodule #{module} do + use Rustler, + otp_app: :rustler, + path: #{inspect(crate_path)}, + skip_compilation?: true, + lib: false + + ~CARGO\""" + [package] + name = "sigil_test" + \""" + + ~CARGO\""" + [lib] + crate-type = ["cdylib"] + \""" + + ~RUST\""" + fn first() {} + \""" + + ~RUST\""" + fn second() {} + \""" + end + """) + + cargo = File.read!(Path.join(crate_path, "Cargo.toml")) + rust = File.read!(Path.join([crate_path, "src", "lib.rs"])) + + assert cargo =~ "[package]" + assert cargo =~ ~s(name = "sigil_test") + assert cargo =~ "[lib]" + assert :binary.match(rust, "fn first()") < :binary.match(rust, "fn second()") + end +end diff --git a/rustler_tests/lib/dynamic_data.ex b/rustler_tests/lib/dynamic_data.ex index f61bd9122..5724641b1 100644 --- a/rustler_tests/lib/dynamic_data.ex +++ b/rustler_tests/lib/dynamic_data.ex @@ -9,6 +9,4 @@ defmodule DynamicData do otp_app: :rustler_test, crate: :dynamic_load, load_data_fun: {DynamicData.Config, :nif_data} - - def get_dataset, do: :erlang.nif_error(:nif_not_loaded) end diff --git a/rustler_tests/lib/resource_dyncall.ex b/rustler_tests/lib/resource_dyncall.ex index d642ad9de..fc08d6894 100644 --- a/rustler_tests/lib/resource_dyncall.ex +++ b/rustler_tests/lib/resource_dyncall.ex @@ -3,9 +3,5 @@ if RustlerTest.Helper.has_nif_version("2.16") do use Rustler, otp_app: :rustler_test, crate: :resource_dyncall - - def new(_), do: err() - - defp err(), do: :erlang.nif_error(:nif_not_loaded) end end diff --git a/rustler_tests/lib/rustler_bigint_test.ex b/rustler_tests/lib/rustler_bigint_test.ex index b02ce2b20..1df1af631 100644 --- a/rustler_tests/lib/rustler_bigint_test.ex +++ b/rustler_tests/lib/rustler_bigint_test.ex @@ -2,12 +2,4 @@ defmodule RustlerBigintTest do use Rustler, otp_app: :rustler_test, crate: :rustler_bigint_test - - defp err do - throw(NifNotLoadedError) - end - - def echo(_), do: err() - def add_one(_), do: err() - def add(_, _), do: err() end diff --git a/rustler_tests/lib/rustler_test.ex b/rustler_tests/lib/rustler_test.ex index 220072b4f..f86d42d58 100644 --- a/rustler_tests/lib/rustler_test.ex +++ b/rustler_tests/lib/rustler_test.ex @@ -21,34 +21,6 @@ defmodule RustlerTest do crate: :rustler_test, features: [Helper.nif_feature_from_running_version()] - defp err, do: :erlang.nif_error(:nif_not_loaded) - - def add_u32(_, _), do: err() - def add_i32(_, _), do: err() - def add_floats(_, _), do: err() - def echo_u8(_), do: err() - def echo_i8(_), do: err() - def echo_u16(_), do: err() - def echo_i16(_), do: err() - def echo_char(_), do: err() - def echo_non_zero_u8(_), do: err() - def echo_non_zero_i64(_), do: err() - def echo_u128(_), do: err() - def echo_i128(_), do: err() - def option_inc(_), do: err() - def erlang_option_inc(_), do: err() - def result_to_int(_), do: err() - - def sum_list(_), do: err() - def make_list(), do: err() - def sum_list_as_floats(_), do: err() - - def compare_local_pids(_, _), do: err() - def are_equal_local_pids(_, _), do: err() - def pid_hash(_), do: err() - - def term_debug(_), do: err() - def term_debug_and_reparse(term) do with debug_str <- term_debug(term), debug_str <- :erlang.binary_to_list(debug_str) ++ ~c".", @@ -58,128 +30,4 @@ defmodule RustlerTest do res end end - - def term_eq(_, _), do: err() - def term_cmp(_, _), do: err() - def term_internal_hash(_, _), do: err() - def term_phash2_hash(_), do: err() - def term_type(_term), do: err() - - def sum_map_values(_), do: err() - def map_entries(_), do: err() - def map_entries_reversed(_), do: err() - def map_from_arrays(_keys, _values), do: err() - def map_from_pairs(_pairs), do: err() - def map_from_macro(), do: err() - def map_generic(_), do: err() - def map_atom_keys(_), do: err() - - def resource_make(), do: err() - def resource_set_integer_field(_, _), do: err() - def resource_get_integer_field(_), do: err() - def resource_make_immutable(_), do: err() - def resource_immutable_count(), do: err() - - def monitor_resource_make(), do: err() - def monitor_resource_monitor(_, _), do: err() - def monitor_resource_down_called(_), do: err() - def monitor_resource_demonitor(_), do: err() - - def resource_make_with_binaries(), do: err() - def resource_make_binaries(_), do: err() - - def make_shorter_subbinary(_), do: err() - def parse_integer(_), do: err() - def binary_new(), do: err() - def owned_binary_new(), do: err() - def new_binary_new(), do: err() - def new_binary_from_iter(), do: err() - def owned_binary_from_iter(), do: err() - def unowned_to_owned(_), do: err() - def realloc_shrink(), do: err() - def realloc_grow(), do: err() - def encode_string(), do: err() - def decode_iolist(_), do: err() - def first_four_bytes_of_iolist(_), do: err() - - def array_into_binary(), do: err() - def vec_into_binary(), do: err() - def slice_into_binary(), do: err() - def str_into_binary(), do: err() - def string_into_binary(), do: err() - - def binary_from_str(), do: err() - - def atom_to_string(_), do: err() - def atom_equals_ok(_), do: err() - def binary_to_atom(_), do: err() - def binary_to_atom_utf8(_), do: err() - def binary_to_existing_atom(_), do: err() - def binary_to_existing_atom_utf8(_), do: err() - - def threaded_fac(_), do: err() - def threaded_sleep(_), do: err() - - def send_all(_, _), do: err() - def send(_, _), do: err() - def whereis_pid(_), do: err() - def is_process_alive(_), do: err() - def sublists(_), do: err() - def make_refs(), do: err() - - def tuple_echo(_), do: err() - def record_echo(_), do: err() - def map_echo(_), do: err() - def exception_echo(_), do: err() - def struct_echo(_), do: err() - def unit_enum_echo(_), do: err() - def tagged_enum_1_echo(_), do: err() - def tagged_enum_2_echo(_), do: err() - def tagged_enum_3_echo(_), do: err() - def tagged_enum_4_echo(_), do: err() - def untagged_enum_echo(_), do: err() - def untagged_enum_with_truthy(_), do: err() - def untagged_enum_for_issue_370(_), do: err() - def newtype_echo(_), do: err() - def tuplestruct_echo(_), do: err() - def newtype_record_echo(_), do: err() - def tuplestruct_record_echo(_), do: err() - def reserved_keywords_type_echo(_), do: err() - def generic_struct_echo(_), do: err() - def mk_generic_map(_), do: err() - - def dirty_io(), do: err() - def dirty_cpu(), do: err() - - def sum_range(_), do: err() - - def bad_arg_error(), do: err() - def atom_str_error(), do: err() - def raise_atom_error(), do: err() - def raise_term_with_string_error(), do: err() - def raise_term_with_atom_error(), do: err() - def term_with_tuple_error(), do: err() - - def nif_attrs_can_rename!(), do: err() - - def add_from_tuple(_tuple), do: err() - def add_one_to_tuple(_tuple), do: err() - def join_tuple_elements(_tuple), do: err() - def maybe_add_one_to_tuple(_tuple), do: err() - def add_i32_from_tuple(_tuple), do: err() - def greeting_person_from_tuple(_tuple), do: err() - - def append_to_path(_path, _to_append), do: err() - - def panic_in_nif(), do: err() - def panic_in_encode(), do: err() - def panic_in_decode(_), do: err() - - if Helper.has_nif_version("2.16") do - def perform_dyncall(_res, _a, _b, _c), do: err() - end - - if Helper.has_nif_version("2.18") do - def term_size(_term), do: err() - end end