Skip to content

Modernise, update to Ort 2.0.0-rc.11 #47

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
serpent213 wants to merge 11 commits into
base: main
Choose a base branch
from
+1,012 −810
Open

Modernise, update to Ort 2.0.0-rc.11 #47

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
b6b1045
refactor/fix: Modernise Python export_resnet helper
serpent213 Jan 30, 2026
550bb20
fix: quiet rust warnings in NIF resource registration
serpent213 Jan 30, 2026
ad731ac
nif: harden tensor IO and error paths
serpent213 Jan 30, 2026
ff4bfc0
deps update: align rustler + refresh locks
serpent213 Jan 30, 2026
9a9b8f2
deps update: ort rc.11 + nif compat fixes
serpent213 Jan 30, 2026
d6a17cd
robustness: tighten NIF + serving defaults
serpent213 Jan 30, 2026
f3159e2
test: correct concat dtype coverage
serpent213 Jan 30, 2026
621ef03
nif: allow skipping compile via env var
serpent213 Jan 30, 2026
c0bae37
release: bump to 0.2.0-rc.1
serpent213 Jan 30, 2026
078ed1a
docs: expand packaging guidance
serpent213 Jan 30, 2026
7d16952
nif: quiet skip-compile and libonnxruntime warnings
serpent213 Jan 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 46 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,54 @@ iex> result |> Nx.backend_transfer() |> Nx.argmax(axis: 1)
```elixir
def deps do
[
{:ortex, "~> 0.1.10"}
{:ortex, "~> 0.2.0-rc.1"}
]
end
```

You will need [Rust](https://www.rust-lang.org/tools/install) for compilation to succeed.

## Execution provider features

Ortex relies on `ort` cargo features to compile support for non-CPU execution providers.
Defaults are OS-specific:

- macOS: `coreml`
- Windows: `directml`
- Linux: none (CPU-only)

Override via `ORTEX_FEATURES` as a comma-separated list. For example:

```sh
ORTEX_FEATURES=cuda,tensorrt mix compile
```

Enabling GPU providers requires the relevant system toolchains to be installed.

### Packaging and Offline Builds

If you are packaging Ortex with a precompiled NIF, set `ORTEX_SKIP_COMPILE=1` during
compilation to avoid building the Rust crate. Ensure the NIF (and any required
`libonnxruntime` binaries) are available in `priv/native` for the target platform.

```sh
ORTEX_SKIP_COMPILE=1 mix compile
```

For offline or system-provided ONNX Runtime builds, you can disable downloads and
link dynamically using a local runtime install:

```sh
ORTEX_SKIP_DOWNLOAD=1 \
ORT_PREFER_DYNAMIC_LINK=1 \
ORT_LIB_LOCATION=/path/to/onnxruntime/lib \
mix compile
```

If your package provides `libonnxruntime.pc`, enable pkg-config lookup:

```sh
ORTEX_FEATURES=pkg-config \
PKG_CONFIG_PATH=/path/to/onnxruntime/lib/pkgconfig \
mix compile
```
11 changes: 9 additions & 2 deletions config/config.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,18 @@ config :ortex,

# Set the cargo feature flags required to use the matching execution provider
# based on the OS we're running on
ortex_features =
default_ortex_features =
case :os.type() do
{:win32, _} -> ["directml"]
{:unix, :darwin} -> ["coreml"]
{:unix, _} -> ["cuda", "tensorrt"]
{:unix, _} -> []
end

ortex_features =
case System.get_env("ORTEX_FEATURES") do
nil -> default_ortex_features
"" -> []
features -> String.split(features, ",", trim: true)
end

config :ortex, Ortex.Native, features: ortex_features
40 changes: 27 additions & 13 deletions lib/ortex/backend.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,34 +60,38 @@ defmodule Ortex.Backend do

@impl true
def backend_transfer(tensor, backend, opts) do
backend.from_binary(tensor, to_binary(tensor), opts)
end

defp to_binary(%T{data: %{ref: tensor}}) do
# filling the bits and limits with 0 since we aren't using them right now
Ortex.Native.to_binary(tensor, 0, 0)
backend.from_binary(tensor, to_binary(tensor, 0), opts)
end

@impl true
def inspect(%T{} = tensor, inspect_opts) do
limit = if inspect_opts.limit == :infinity, do: :infinity, else: inspect_opts.limit + 1
limit =
case inspect_opts.limit do
:infinity -> Nx.size(tensor)
value -> min(value + 1, Nx.size(tensor))
end

tensor
|> to_binary(min(limit, Nx.size(tensor)))
|> to_binary(limit)
|> then(&Nx.Backend.inspect(tensor, &1, inspect_opts))
|> maybe_add_signature(tensor)
end

@impl true
def slice(out, %T{data: %B{ref: tensor_ref}}, start_indicies, lengths, strides) do
r = Ortex.Native.slice(tensor_ref, start_indicies, lengths, strides)
put_in(out.data, %B{ref: r})
case Ortex.Native.slice(tensor_ref, start_indicies, lengths, strides) do
{:error, msg} -> raise msg
res -> put_in(out.data, %B{ref: res})
end
end

@impl true
def reshape(out, %T{data: %B{ref: ref}}) do
shape = Nx.shape(out) |> Tuple.to_list()
put_in(out.data, %B{ref: Ortex.Native.reshape(ref, shape)})
case Ortex.Native.reshape(ref, shape) do
{:error, msg} -> raise msg
res -> put_in(out.data, %B{ref: res})
end
end

@impl true
Expand All @@ -102,7 +106,14 @@ defmodule Ortex.Backend do
out
| shape: new_shape,
names: new_names,
data: %B{ref: Ortex.Native.reshape(ref, new_shape |> Tuple.to_list())}
data:
%B{
ref:
case Ortex.Native.reshape(ref, new_shape |> Tuple.to_list()) do
{:error, msg} -> raise msg
res -> res
end
}
}
end
end
Expand All @@ -121,7 +132,10 @@ defmodule Ortex.Backend do

type = out.type

%{out | data: %B{ref: Ortex.Native.concatenate(tensor_refs, type, axis)}}
case Ortex.Native.concatenate(tensor_refs, type, axis) do
{:error, msg} -> raise msg
res -> %{out | data: %B{ref: res}}
end
end

if Application.compile_env(:ortex, :add_backend_on_inspect, true) do
Expand Down
34 changes: 26 additions & 8 deletions lib/ortex/native.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,37 @@
defmodule Ortex.Native do
@moduledoc false

@rustler_version Application.spec(:rustler, :vsn) |> to_string() |> Version.parse!()
@skip_compile? (case System.get_env("ORTEX_SKIP_COMPILE") do
nil -> false
value -> String.downcase(value) in ["1", "true", "yes", "on"]
end)
@skip_download? (case System.get_env("ORTEX_SKIP_DOWNLOAD") do
nil -> false
value -> String.downcase(value) in ["1", "true", "yes", "on"]
end)

# We have to compile the crate before `use Rustler` compiles the crate since
# cargo downloads the onnxruntime shared libraries and they are not available
# to load or copy into Elixir's during the on_load or Elixir compile steps.
# In the future, this may be configurable in Rustler.
if Version.compare(@rustler_version, "0.30.0") in [:gt, :eq] do
Rustler.Compiler.compile_crate(:ortex, Application.compile_env(:ortex, __MODULE__, []),
otp_app: :ortex,
crate: :ortex
)
else
Rustler.Compiler.compile_crate(__MODULE__, otp_app: :ortex, crate: :ortex)
if not @skip_compile? do
if @skip_download? do
System.put_env("ORT_SKIP_DOWNLOAD", "1")
end

rustler_version =
Application.spec(:rustler, :vsn)
|> to_string()
|> Version.parse!()

if Version.compare(rustler_version, "0.30.0") in [:gt, :eq] do
Rustler.Compiler.compile_crate(:ortex, Application.compile_env(:ortex, __MODULE__, []),
otp_app: :ortex,
crate: :ortex
)
else
Rustler.Compiler.compile_crate(__MODULE__, otp_app: :ortex, crate: :ortex)
end
end

Ortex.Util.copy_ort_libs()
Expand Down
25 changes: 19 additions & 6 deletions lib/ortex/serving.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,28 @@ defmodule Ortex.Serving do
@behaviour Nx.Serving

@impl true
def init(_inline_or_process, model, [_defn_options]) do
def init(_inline_or_process, model, defn_options) when is_list(defn_options) do
defn_options =
Enum.map(defn_options, fn opts ->
opts = if is_list(opts), do: opts, else: []
Keyword.put_new(opts, :compiler, Nx.Defn.Evaluator)
end)

func = fn x -> Ortex.run(model, x) end
{:ok, func}
{:ok, {func, defn_options}}
end

@impl true
def handle_batch(batch, _partition, function) do
# A hack to move the back into a tensor for Ortex
out = function.(Nx.Defn.jit_apply(&Function.identity/1, [batch]))
{:execute, fn -> {out, :server_info} end, function}
def handle_batch(batch, partition, {function, defn_options}) do
opts = Enum.at(defn_options, partition) || []

materialized =
case batch do
%Nx.Batch{} -> Nx.Defn.jit_apply(&Function.identity/1, [batch], opts)
_ -> batch
end

out = function.(materialized)
{:execute, fn -> {out, :server_info} end, {function, defn_options}}
end
end
127 changes: 109 additions & 18 deletions lib/ortex/util.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,31 +5,122 @@ defmodule Ortex.Util do
Elixir can use
"""
def copy_ort_libs() do
suppress_warning? = suppress_copy_warning?()
build_root = Path.absname(:code.priv_dir(:ortex)) |> Path.dirname()
ort_lib_location = System.get_env("ORT_LIB_LOCATION")
destination_dir = Path.join([:code.priv_dir(:ortex), "native"])
File.mkdir_p!(destination_dir)

rust_env =
case Path.join([build_root, "native/ortex/release"]) |> File.ls() do
{:ok, _} -> "release"
_ -> "debug"
end

# where the libonnxruntime files are stored
rust_path = Path.join([build_root, "native/ortex", rust_env])
search_patterns =
[build_root, find_project_root(build_root), find_project_root(File.cwd!())]
|> Enum.reject(&is_nil/1)
|> Enum.uniq()
|> Enum.flat_map(&patterns_for_root/1)
|> Enum.concat(ort_lib_location_patterns(ort_lib_location))
|> Enum.uniq()

onnx_runtime_paths =
search_patterns
|> Enum.flat_map(&Path.wildcard/1)
|> Enum.uniq()

existing = Path.wildcard(lib_glob(destination_dir))

cond do
onnx_runtime_paths == [] and existing != [] ->
:ok

onnx_runtime_paths == [] and not is_nil(ort_lib_location) ->
raise """
Unable to locate libonnxruntime binaries.
ORT_LIB_LOCATION: #{ort_lib_location}
Searched: #{Enum.join(search_patterns, ", ")}
Destination: #{destination_dir}
"""

onnx_runtime_paths == [] and (test_env?() or suppress_warning?) ->
:ok

onnx_runtime_paths == [] ->
:ok

true ->
Enum.each(onnx_runtime_paths, fn path ->
File.cp!(path, Path.join([destination_dir, Path.basename(path)]))
end)
end
end

defp patterns_for_root(root) do
[
Path.join([root, "native/ortex/release"]),
Path.join([root, "native/ortex/debug"]),
Path.join([root, "native/ortex/target/release"]),
Path.join([root, "native/ortex/target/debug"]),
Path.join([root, "native/ortex/target", "**"])
]
|> Enum.map(&lib_glob/1)
end

defp ort_lib_location_patterns(nil), do: []

defp ort_lib_location_patterns(path) do
expanded = Path.expand(path)

if File.dir?(expanded) do
[expanded, Path.join(expanded, "lib"), Path.join(expanded, "lib64")]
|> Enum.filter(&File.dir?/1)
|> Enum.map(&lib_glob/1)
else
[expanded]
end
end

defp lib_glob(base) do
suffix =
case :os.type() do
{:win32, _} -> Path.join([rust_path, "libonnxruntime*.dll*"])
{:unix, :darwin} -> Path.join([rust_path, "libonnxruntime*.dylib*"])
{:unix, _} -> Path.join([rust_path, "libonnxruntime*.so*"])
{:win32, _} -> "libonnxruntime*.dll*"
{:unix, :darwin} -> "libonnxruntime*.dylib*"
{:unix, _} -> "libonnxruntime*.so*"
end
|> Path.wildcard()

# where we need to copy the paths
destination_dir = Path.join([:code.priv_dir(:ortex), "native"])
Path.join([base, suffix])
end

defp find_project_root(path) do
expanded = Path.expand(path)

cond do
File.exists?(Path.join(expanded, "mix.exs")) ->
expanded

expanded == Path.dirname(expanded) ->
nil

true ->
find_project_root(Path.dirname(expanded))
end
end

defp test_env?() do
cond do
Code.ensure_loaded?(Mix) and function_exported?(Mix, :env, 0) ->
Mix.env() == :test

true ->
System.get_env("MIX_ENV") == "test"
end
end

defp suppress_copy_warning?() do
truthy_env?("ORTEX_SKIP_COMPILE") or truthy_env?("ORTEX_SKIP_DOWNLOAD") or
truthy_env?("ORT_PREFER_DYNAMIC_LINK")
end

onnx_runtime_paths
|> Enum.map(fn x ->
File.cp!(x, Path.join([destination_dir, Path.basename(x)]))
end)
defp truthy_env?(name) do
case System.get_env(name) do
nil -> false
value -> String.downcase(value) in ["1", "true", "yes", "on"]
end
end
end
Loading