mix nif compiler prototype

Author: hansihe

rustler_mix/.gitignore

@@ -0,0 +1,8 @@
+/_build
+/cover
+/deps
+erl_crash.dump
+*.ez
+
+/src/toml_parser.erl
+/src/toml_lexer.erl

rustler_mix/README.md

@@ -0,0 +1,20 @@
+# Rustler
+
+**TODO: Add description**
+
+## Installation
+
+If [available in Hex](https://hex.pm/docs/publish), the package can be installed as:
+
+  1. Add rustler_mix to your list of dependencies in `mix.exs`:
+
+        def deps do
+          [{:rustler_mix, "~> 0.0.1"}]
+        end
+
+  2. Ensure rustler_mix is started before your application:
+
+        def application do
+          [applications: [:rustler_mix]]
+        end
+

rustler_mix/config/config.exs

@@ -0,0 +1,30 @@
+# This file is responsible for configuring your application
+# and its dependencies with the aid of the Mix.Config module.
+use Mix.Config
+
+# This configuration is loaded before any dependency and is restricted
+# to this project. If another project depends on this project, this
+# file won't be loaded nor affect the parent project. For this reason,
+# if you want to provide default values for your application for
+# 3rd-party users, it should be done in your "mix.exs" file.
+
+# You can configure for your application as:
+#
+#     config :rustler_mix, key: :value
+#
+# And access this configuration in your application as:
+#
+#     Application.get_env(:rustler_mix, :key)
+#
+# Or configure a 3rd-party app:
+#
+#     config :logger, level: :info
+#
+
+# It is also possible to import configuration files, relative to this
+# directory. For example, you can emulate configuration per environment
+# by uncommenting the line below and defining dev.exs, test.exs and such.
+# Configuration from the imported file will override the ones defined
+# here (which is why it is important to import them last).
+#
+#     import_config "#{Mix.env}.exs"

rustler_mix/lib/compiler.ex

@@ -0,0 +1,128 @@
+defmodule Mix.Tasks.Compile.Rustler do
+  use Mix.Task
+
+  alias Rustler.Compiler.Messages
+  alias Rustler.Compiler.Multirust
+
+  def throw_error(error_descr) do
+    Mix.shell.error Messages.message(error_descr)
+    raise "Compilation error"
+  end
+
+  def run(_args) do
+    project = Mix.Project.config
+    crates = project[:rustler_crates]
+
+    if Multirust.version == :none do
+      throw_error(:multirust_not_installed)
+    end
+
+    rust_versions = Multirust.rust_versions
+    unless Rustler.rust_version in rust_versions do
+      throw_error({:rust_version_not_installed, Rustler.rust_version})
+    end
+
+    nif_version = :erlang.system_info(:nif_version)
+    unless nif_version in Rustler.nif_versions do
+      throw_error({:unsupported_nif_version, nif_version})
+    end
+
+    File.mkdir_p!(Rustler.nif_lib_dir)
+
+    is_release = System.get_env("RUSTLER_DEBUG") != "true"
+    compile_crates(crates, is_release)
+  end
+
+  def compile_crates(nil, _), do: throw_error(:no_crates_property)
+  def compile_crates(crates, release) do
+    Enum.map(crates, &compile_crate(&1, release))
+  end
+
+  def compile_crate(crate, release) do
+    cargo_data = check_crate_env(crate)
+
+    lib_name = Rustler.TomlParser.get_table_val(cargo_data, ["lib"], "name")
+    if lib_name == nil do
+      throw_error({:cargo_no_library, crate})
+    end
+
+    compiler_flags = if release, do: ["--release"], else: []
+    compile_flavor = if release, do: "release", else: "debug"
+
+    target_dir = "#{Rustler.nif_lib_dir}/#{crate}_target"
+
+    Mix.shell.info "Compiling NIF crate '#{crate}' (#{compile_flavor})..."
+    case Multirust.compile_with(crate, Rustler.rust_version, compiler_flags,
+                                [{"CARGO_TARGET_DIR", target_dir}],
+                                IO.stream(:stdio, :line)) do
+      {_, 0} -> nil
+      _ -> raise "Compile error"
+    end
+
+    # TODO: Make build flavor specific
+    Enum.map(Path.wildcard("#{target_dir}/#{compile_flavor}/lib#{lib_name}.*"), fn
+      path ->
+        file_name = Path.basename(path)
+        File.cp!(Path.absname(path), "#{Rustler.nif_lib_dir}/#{file_name}")
+    end)
+  end
+
+  def check_crate_env(crate) do
+    unless File.dir?(crate) do
+      Mix.shell.error Messages.message(:no_crates_property_message)
+    end
+
+    cargo_data =
+      case File.read("#{crate}/Cargo.toml") do
+        {:error, :enoent} ->
+          throw_error({:cargo_toml_not_found, crate})
+        {:ok, text} ->
+          Rustler.TomlParser.parse(text)
+      end
+
+    rustler_version = get_cargo_dep_info(cargo_data, "rustler")
+    rustler_codegen_version = get_cargo_dep_info(cargo_data, "rustler_codegen")
+
+    validate_versions(crate, rustler_version, rustler_codegen_version)
+
+    cargo_data
+  end
+
+  def validate_versions(crate, rustler, codegen)
+  when rustler == nil or codegen == nil do
+    throw_error({:no_rustler_deps, crate, Rustler.rustler_version})
+  end
+  def validate_versions(_, %{version: nil}, %{version: nil}) do
+    # Development, no validation
+  end
+  def validate_versions(crate, %{version: rustler_v}, %{version: codegen_v})
+  when rustler_v != codegen_v do
+    throw_error({:differing_versions, crate, rustler_v, codegen_v})
+  end
+  def validate_versions(crate, %{version: version}, %{version: version}) do
+    unless version == "=#{Rustler.rustler_version}" do
+      throw_error({:unsupported_rustler_version, crate,
+                   "=#{Rustler.rustler_version}", version})
+    end
+  end
+
+  def get_cargo_dep_info(data, dep_name) do
+    table_path = ["dependencies", dep_name]
+    simple_key = Rustler.TomlParser.get_table_val(data, ["dependencies"], dep_name)
+    table = Rustler.TomlParser.get_table_vals(data, ["dependencies", dep_name])
+
+    cond do
+      simple_key != nil ->
+        %{version: simple_key}
+      table != nil ->
+        %{
+          version: Rustler.TomlParser.get_keys_key(table, "version"),
+          path: Rustler.TomlParser.get_keys_key(table, "path"),
+          git: Rustler.TomlParser.get_keys_key(table, "git"),
+        }
+      true ->
+        nil
+    end
+  end
+
+end

rustler_mix/lib/compiler/messages.ex

@@ -0,0 +1,80 @@
+defmodule Rustler.Compiler.Messages do
+
+  def message(:multirust_not_installed) do
+    """
+    Multirust could not be found on your machine.
+    It is required for building NIF crates automatically.
+
+    Follow the instructions in https://github.com/brson/multirust to install it.
+    """
+  end
+
+  def message({:nonexistent_crate_directory, crate}) do
+    """
+    Directory '#{crate}' could not be found.
+    One of the crate directories specified for compilation does not exist.
+    """
+  end
+
+  def message({:cargo_toml_not_found, crate}) do
+    """
+    No 'Cargo.toml' could be found in the '#{crate}' directory.
+    Are you sure it is a Rust crate?
+    """
+  end
+
+  def message(:no_crates_property) do
+    """
+    No NIF crates listed in project definition.
+    Add some crate directories under :rustler_crates in the mix.exs project definition to compile them.
+
+    Example:
+    rustler_crates: ["my_nif_crate/"]
+    """
+  end
+
+  def message({:differing_versions, crate, rustler_version, codegen_version}) do
+    """
+    The '#{crate}' crate should have the same rustler and rustler_codegen version in its Cargo.toml:
+    rustler version: #{rustler_version}
+    rustler_codegen version: #{codegen_version}
+    """
+  end
+
+  def message({:unsupported_rustler_version, crate, supported, version}) do
+    """
+    The crate '#{crate}' is using an unsupported version of rustler and rustler_codegen.
+
+    Should be: '#{supported}''
+    Is: '#{version}'
+
+    Note: The version numbers need to match exactly, including space and equals.
+    """
+  end
+
+  def message({:rust_version_not_installed, needed_version}) do
+    """
+    Required Rust toolchain version #{needed_version} is not installed.
+    It can be installed by running 'multirust update #{needed_version}'.
+    """
+  end
+
+  def message({:no_rustler_deps, crate, version}) do
+    """
+    No Rustler dependencies found in Cargo.toml for crate #{crate}.
+    Please make sure the following is in the dependencies section of your Cargo.toml file:
+
+    > rustler = #{version}
+    > rustler_codegen = #{version}
+
+    Note: You should already have this if you made your project with the project generator.
+    """
+  end
+
+  def message({:cargo_no_library, crate}) do
+    """
+    No library with name listed in Cargo.toml of crate '#{crate}'.
+    """
+  end
+
+end

rustler_mix/lib/compiler/multirust.ex

@@ -0,0 +1,30 @@
+defmodule Rustler.Compiler.Multirust do
+
+  def version do
+    multirust_exec = System.find_executable("multirust")
+    if multirust_exec == nil do
+      :none
+    else
+      {version, 0} = System.cmd("multirust", ["--version"])
+      {:ok, version}
+    end
+  end
+
+  def rust_versions do
+    {versions_raw, 0} = System.cmd("multirust", ["list-toolchains"])
+    versions_raw
+    |> String.strip(?\n)
+    |> String.split("\n")
+  end
+
+  def install_toolchain(version) do
+    {_resp, 0} = System.cmd("multirust", ["update", version],
+                            into: IO.stream(:stdio, :line))
+  end
+
+  def compile_with(path, version, args \\ [], env \\ [], into \\ "") do
+    System.cmd("multirust", ["run", version, "cargo", "build" | args],
+               cd: path, into: into, env: env)
+  end
+
+end

rustler_mix/lib/rustler.ex

@@ -0,0 +1,26 @@
+defmodule Rustler do
+
+  def rustler_version, do: "0.8.1"
+  def rust_version, do: "nightly-2016-04-05"
+  def nif_versions, do: [
+    '2.7',
+    '2.8',
+    '2.9',
+    '2.10',
+  ]
+
+  def nif_lib_dir do
+    Mix.Project.build_path <> "/rustler"
+  end
+
+  def nif_lib_path(lib_name) do
+    "#{nif_lib_dir}/lib#{lib_name}"
+  end
+
+  defmacro load_nif(lib_name, load_args \\ nil) do
+    quote do
+      :erlang.load_nif(Rustler.nif_lib_path(unquote(lib_name)), unquote(load_args))
+    end
+  end
+
+end