From 0293f723e5f1751790bf44f82ba73c55b7ab0370 Mon Sep 17 00:00:00 2001 From: "Matthieu C." <886074+teto@users.noreply.github.com> Date: Wed, 11 Jun 2025 19:05:04 +0200 Subject: [PATCH 1/5] adding crane --- flake.nix | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/flake.nix b/flake.nix index 4a97ee0..b1a618b 100644 --- a/flake.nix +++ b/flake.nix @@ -6,12 +6,14 @@ url = "github:oxalica/rust-overlay"; inputs.nixpkgs.follows = "nixpkgs"; }; + crane.url = "github:ipetkov/crane"; }; outputs = { nixpkgs, flake-utils, rust-overlay, + crane, ... }: flake-utils.lib.eachDefaultSystem ( @@ -20,8 +22,19 @@ overlays = [ (import rust-overlay) ]; pkgs = import nixpkgs { inherit system overlays; }; rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; + + craneLib = crane.mkLib pkgs; in { + packages.default = craneLib.buildPackage { + src = craneLib.cleanCargoSource ./.; + + # Add extra inputs here or any other derivation settings + # doCheck = true; + # buildInputs = []; + # nativeBuildInputs = []; + }; + devShells.default = pkgs.mkShell { buildInputs = with pkgs; [ just From 79933592440d7269fa48eba803f217bc7c977024 Mon Sep 17 00:00:00 2001 From: "Matthieu C." <886074+teto@users.noreply.github.com> Date: Wed, 11 Jun 2025 19:05:13 +0200 Subject: [PATCH 2/5] adding crane to lock --- flake.lock | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/flake.lock b/flake.lock index b679ea3..39e5c8b 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,20 @@ { "nodes": { + "crane": { + "locked": { + "lastModified": 1748970125, + "narHash": "sha256-UDyigbDGv8fvs9aS95yzFfOKkEjx1LO3PL3DsKopohA=", + "owner": "ipetkov", + "repo": "crane", + "rev": "323b5746d89e04b22554b061522dfce9e4c49b18", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -36,6 +51,7 @@ }, "root": { "inputs": { + "crane": "crane", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", "rust-overlay": "rust-overlay" From 08d5baf514201960af4d2d32435346f7892f5b71 Mon Sep 17 00:00:00 2001 From: "Matthieu C." <886074+teto@users.noreply.github.com> Date: Thu, 12 Jun 2025 12:13:00 +0200 Subject: [PATCH 3/5] TODO fix lindera build --- flake.lock | 48 ++++++------- flake.nix | 206 ++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 212 insertions(+), 42 deletions(-) diff --git a/flake.lock b/flake.lock index 39e5c8b..0453752 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,21 @@ { "nodes": { + "advisory-db": { + "flake": false, + "locked": { + "lastModified": 1748950236, + "narHash": "sha256-kNiGMrXi5Bq/aWoQmnpK0v+ufQA4FOInhbkY56iUndc=", + "owner": "rustsec", + "repo": "advisory-db", + "rev": "a1f651cba8bf224f52c5d55d8182b3bb0ebce49e", + "type": "github" + }, + "original": { + "owner": "rustsec", + "repo": "advisory-db", + "type": "github" + } + }, "crane": { "locked": { "lastModified": 1748970125, @@ -35,46 +51,26 @@ }, "nixpkgs": { "locked": { - "lastModified": 1747542820, - "narHash": "sha256-GaOZntlJ6gPPbbkTLjbd8BMWaDYafhuuYRNrxCGnPJw=", + "lastModified": 1749558678, + "narHash": "sha256-DUVAe8E2X2QM0dAnTGlTiqemMqUMMyIeCH7UeNo0g64=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "292fa7d4f6519c074f0a50394dbbe69859bb6043", + "rev": "a12f3a99614894502e73eb816e9e076b0ab05730", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "nixpkgs-unstable", "repo": "nixpkgs", "type": "github" } }, "root": { "inputs": { + "advisory-db": "advisory-db", "crane": "crane", "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs", - "rust-overlay": "rust-overlay" - } - }, - "rust-overlay": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1747708620, - "narHash": "sha256-eqQ6D9o7WUpwarjmkzW/20bfqmhhKqGgPOhDdvJddxw=", - "owner": "oxalica", - "repo": "rust-overlay", - "rev": "3e7b002daad1ff342b223af3a9de7b2a4b6fdc7d", - "type": "github" - }, - "original": { - "owner": "oxalica", - "repo": "rust-overlay", - "type": "github" + "nixpkgs": "nixpkgs" } }, "systems": { diff --git a/flake.nix b/flake.nix index b1a618b..6903016 100644 --- a/flake.nix +++ b/flake.nix @@ -1,40 +1,207 @@ { + description = "Build a cargo workspace"; + inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + crane.url = "github:ipetkov/crane"; + flake-utils.url = "github:numtide/flake-utils"; - rust-overlay = { - url = "github:oxalica/rust-overlay"; - inputs.nixpkgs.follows = "nixpkgs"; + + advisory-db = { + url = "github:rustsec/advisory-db"; + flake = false; }; - crane.url = "github:ipetkov/crane"; }; + outputs = { + self, nixpkgs, - flake-utils, - rust-overlay, crane, + flake-utils, + advisory-db, ... }: flake-utils.lib.eachDefaultSystem ( system: let - overlays = [ (import rust-overlay) ]; - pkgs = import nixpkgs { inherit system overlays; }; - rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; + pkgs = nixpkgs.legacyPackages.${system}; + inherit (pkgs) lib; + + # the lindera-unidic v0.32.2 crate uses [1] an outdated unidic-mecab fork [2] and builds it in pure rust + # [1] https://github.com/lindera/lindera/blob/v0.32.2/lindera-unidic/build.rs#L5-L11 + # [2] https://github.com/lindera/unidic-mecab + lindera-unidic-src = pkgs.fetchurl { + url = "https://dlwqk3ibdg1xh.cloudfront.net/unidic-mecab-2.1.2.tar.gz"; + hash = "sha256-JKx1/k5E2XO1XmWEfDX6Suwtt6QaB7ScoSUUbbn8EYk="; + }; craneLib = crane.mkLib pkgs; + src = craneLib.cleanCargoSource ./.; + + # Common arguments can be set here to avoid repeating them later + commonArgs = { + inherit src; + strictDeps = true; + + buildInputs = + [ + # Add additional build inputs here + ] + ++ lib.optionals pkgs.stdenv.isDarwin [ + # Additional darwin specific inputs can be set here + pkgs.libiconv + ]; + + # Additional environment variables can be set directly + # MY_CUSTOM_VAR = "some value"; + # LINDERA_CACHE="${lindera-unidic-src}"; + }; + + # Build *just* the cargo dependencies (of the entire workspace), + # so we can reuse all of that work (e.g. via cachix) when running in CI + # It is *highly* recommended to use something like cargo-hakari to avoid + # cache misses when building individual top-level-crates + cargoArtifacts = craneLib.buildDepsOnly commonArgs; + + individualCrateArgs = commonArgs // { + inherit cargoArtifacts; + inherit (craneLib.crateNameFromCargoToml { inherit src; }) version; + # NB: we disable tests since we'll run them all via cargo-nextest + doCheck = false; + }; + + fileSetForCrate = + crate: + lib.fileset.toSource { + root = ./.; + fileset = lib.fileset.unions [ + ./Cargo.toml + ./Cargo.lock + (lib.fileset.fileFilter (file: file.hasExt "md") ./.) + (craneLib.fileset.commonCargoSources ./crates/wordbase) + ./crates/wordbase-cli + (craneLib.fileset.commonCargoSources ./crates/jmdict-furigana) + ./crates/jmdict-furigana/README.md + ./crates/jmdict-furigana/src/jmdict_furigana.json.zip + (craneLib.fileset.commonCargoSources ./crates/wordbase-api) + ./crates/wordbase/src/records.html + ./crates/wordbase/README.md + ./crates/wordbase/migrations + ./crates/wordbase-api/README.md + ./.sqlx + + # (craneLib.fileset.commonCargoSources ./crates/my-workspace-hack) + (craneLib.fileset.commonCargoSources crate) + ]; + }; + + # Build the top-level crates of the workspace as individual derivations. + # This allows consumers to only depend on (and build) only what they need. + # Though it is possible to build the entire workspace as a single derivation, + # so this is left up to you on how to organize things + # + # Note that the cargo workspace must define `workspace.members` using wildcards, + # otherwise, omitting a crate (like we do below) will result in errors since + # cargo won't be able to find the sources for all members. + wordbase-cli = craneLib.buildPackage ( + individualCrateArgs + // { + pname = "wordbase-cli"; + cargoExtraArgs = "-p wordbase-cli"; + src = fileSetForCrate ./crates/wordbase-cli; + } + ); + + # my-server = craneLib.buildPackage ( + # individualCrateArgs + # // { + # pname = "my-server"; + # cargoExtraArgs = "-p my-server"; + # src = fileSetForCrate ./crates/my-server; + # } + # ); in { - packages.default = craneLib.buildPackage { - src = craneLib.cleanCargoSource ./.; + checks = { + # Build the crates as part of `nix flake check` for convenience + inherit wordbase-cli; + + # Run clippy (and deny all warnings) on the workspace source, + # again, reusing the dependency artifacts from above. + # + # Note that this is done as a separate derivation so that + # we can block the CI if there are issues here, but not + # prevent downstream consumers from building our crate by itself. + # my-workspace-clippy = craneLib.cargoClippy ( + # commonArgs + # // { + # inherit cargoArtifacts; + # cargoClippyExtraArgs = "--all-targets -- --deny warnings"; + # } + # ); + + wordbase-workspace-doc = craneLib.cargoDoc ( + commonArgs + // { + inherit cargoArtifacts; + } + ); + + # Check formatting + wordbase-workspace-fmt = craneLib.cargoFmt { + inherit src; + }; - # Add extra inputs here or any other derivation settings - # doCheck = true; - # buildInputs = []; - # nativeBuildInputs = []; + my-workspace-toml-fmt = craneLib.taploFmt { + src = pkgs.lib.sources.sourceFilesBySuffices src [ ".toml" ]; + # taplo arguments can be further customized below as needed + # taploExtraArgs = "--config ./taplo.toml"; + }; + + # Ensure that cargo-hakari is up to date + # my-workspace-hakari = craneLib.mkCargoDerivation { + # inherit src; + # pname = "my-workspace-hakari"; + # cargoArtifacts = null; + # doInstallCargoArtifacts = false; + # + # buildPhaseCargoCommand = '' + # cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date + # cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack + # cargo hakari verify + # ''; + # + # nativeBuildInputs = [ + # pkgs.cargo-hakari + # ]; + # }; + }; + + packages = { + inherit wordbase-cli; + }; + + apps = { + wordbase-cli = flake-utils.lib.mkApp { + drv = wordbase-cli; + }; }; + # devShells.default = craneLib.devShell { + # # Inherit inputs from checks. + # checks = self.checks.${system}; + # + # # Additional dev-shell environment variables can be set directly + # # MY_CUSTOM_DEVELOPMENT_VAR = "something else"; + # + # # Extra inputs can be added here; cargo and rustc are provided by default. + # packages = [ + # pkgs.cargo-hakari + # ]; + # }; + devShells.default = pkgs.mkShell { buildInputs = with pkgs; [ just @@ -59,7 +226,14 @@ # Binding generation ktlint ]; + + # shellHook = '' + # # Caching for lindera-unidic + # [ "''${LINDERA_CACHE+x}" ] || + # export LINDERA_CACHE="''${XDG_CACHE_HOME:-$HOME/.cache}/lindera" + # ''; }; + } ); } From d06c07d11f4940f33500270e535e30edf7470fe2 Mon Sep 17 00:00:00 2001 From: "Matthieu C." <886074+teto@users.noreply.github.com> Date: Sun, 15 Jun 2025 01:31:00 +0200 Subject: [PATCH 4/5] wip --- flake.nix | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 6903016..56bce35 100644 --- a/flake.nix +++ b/flake.nix @@ -40,9 +40,65 @@ craneLib = crane.mkLib pkgs; src = craneLib.cleanCargoSource ./.; + baseArgs = { + src = craneLib.cleanCargoSource ./.; + # src = fileSetForCrate; + }; + + # cargoVendorDir + + # isLinderaRepo = p: lib.hasPrefix + # "git+https://github.com/seanmonstar/num_cpus.git" + # p.source; + httplz_port = "34567"; + cargoVendorDir = craneLib.vendorCargoDeps (baseArgs // { + # Use this function to override crates coming from git dependencies + # overrideVendorGitCheckout = ps: drv: + # # For example, patch a specific repository and tag, in this case num_cpus-1.13.1 + # # if lib.any (p: (isLinderaRepo p) ) ps then + # # drv.overrideAttrs (_old: { + # # }) + # # else + # # Nothing to change, leave the derivations as is + # drv; + + + # Use this function to override crates coming from any registry checkout + overrideVendorCargoPackage = p: drv: + # For example, patch a specific crate, in this case byteorder-1.5.0 + if p.name == "lindera-unidic" then + builtins.trace "MATCHED LINDERA-UNIDIC" drv.overrideAttrs(_old: { + CARGO_BUILD_RUSTFLAGS = "--verbose"; + + # Specifying an arbitrary patch to apply + # patches = [ + # ./0001-patch-num-cpus.patch + # ]; + + # Similarly we can also run additional hooks to make changes + # lindera = { version = "0.43.1" } + + postPatch = '' + echo "PATCHING" + ls -l + cat build.rs + substituteInPlace build.rs --replace-fail \ + "https://Lindera.dev/unidic-mecab-2.1.2.tar.gz" \ + "http://localhost:${httplz_port}/unidic-mecab-2.1.2.tar.gz" + + cat build.rs + + ''; + }) + else + # Nothing to change, leave the derivations as is + drv; + }); + # Common arguments can be set here to avoid repeating them later commonArgs = { inherit src; + inherit cargoVendorDir; strictDeps = true; buildInputs = @@ -63,7 +119,39 @@ # so we can reuse all of that work (e.g. via cachix) when running in CI # It is *highly* recommended to use something like cargo-hakari to avoid # cache misses when building individual top-level-crates - cargoArtifacts = craneLib.buildDepsOnly commonArgs; + cargoArtifacts = (craneLib.buildDepsOnly commonArgs).overrideAttrs(oa: { + pname = builtins.trace "pname=${oa.pname}" oa.pname; + + nativeBuildInputs = oa.nativeBuildInputs ++ [ pkgs.httplz]; + doCheck = false; + preBuild = '' + echo "PATCHING" + ls -l + set -x + ( + set -x + # serve lindera-unidic on localhost vacant port + httplz_port="${ + if pkgs.stdenv.buildPlatform.isDarwin then + ''$(python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')'' + else + "34567" + }" + mkdir .lindera-http-plz + ln -s ${lindera-unidic-src} .lindera-http-plz/unidic-mecab-2.1.2.tar.gz + httplz --port "$httplz_port" -- .lindera-http-plz/ & + echo $! >$TMPDIR/.httplz_pid + + echo "$out" + echo "Trying to substitute" + ls -l + + # not needed with useFetchCargoVendor=true, but kept in case it is required again + #newHash=$(sha256sum build.rs | cut -d " " -f 1) + #substituteInPlace .cargo-checksum.json --replace-fail $oldHash $newHash + )''; + + }); individualCrateArgs = commonArgs // { inherit cargoArtifacts; @@ -215,8 +303,8 @@ nixfmt-tree # Rust - rustToolchain - taplo + # rustToolchain + # taplo cargo-shear pkg-config openssl From 3a558d19e7a9bfafd12323fc5271f2f1435bcd8d Mon Sep 17 00:00:00 2001 From: "Matthieu C." <886074+teto@users.noreply.github.com> Date: Sun, 15 Jun 2025 18:04:35 +0200 Subject: [PATCH 5/5] tries to rebuild lindera during wordbase-cli build --- flake.nix | 46 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/flake.nix b/flake.nix index 56bce35..072a7b0 100644 --- a/flake.nix +++ b/flake.nix @@ -124,7 +124,9 @@ nativeBuildInputs = oa.nativeBuildInputs ++ [ pkgs.httplz]; doCheck = false; - preBuild = '' + buildPhase = + # preBuild = + '' echo "PATCHING" ls -l set -x @@ -149,7 +151,11 @@ # not needed with useFetchCargoVendor=true, but kept in case it is required again #newHash=$(sha256sum build.rs | cut -d " " -f 1) #substituteInPlace .cargo-checksum.json --replace-fail $oldHash $newHash - )''; + ) + '' + + oa.buildPhase + # + oa.buildPhaseCargoCommand + ; }); @@ -162,6 +168,7 @@ fileSetForCrate = crate: + # "./${crate}" lib.fileset.toSource { root = ./.; fileset = lib.fileset.unions [ @@ -169,21 +176,22 @@ ./Cargo.lock (lib.fileset.fileFilter (file: file.hasExt "md") ./.) (craneLib.fileset.commonCargoSources ./crates/wordbase) - ./crates/wordbase-cli (craneLib.fileset.commonCargoSources ./crates/jmdict-furigana) ./crates/jmdict-furigana/README.md ./crates/jmdict-furigana/src/jmdict_furigana.json.zip (craneLib.fileset.commonCargoSources ./crates/wordbase-api) - ./crates/wordbase/src/records.html - ./crates/wordbase/README.md - ./crates/wordbase/migrations - ./crates/wordbase-api/README.md - ./.sqlx + (craneLib.fileset.commonCargoSources ./crates/wordbase-sys) + # ./crates/wordbase/src/records.html + # ./crates/wordbase/README.md + # ./crates/wordbase/migrations + # ./crates/wordbase-api/README.md + # ./.sqlx # (craneLib.fileset.commonCargoSources ./crates/my-workspace-hack) (craneLib.fileset.commonCargoSources crate) ]; - }; + } + ; # Build the top-level crates of the workspace as individual derivations. # This allows consumers to only depend on (and build) only what they need. @@ -196,6 +204,7 @@ wordbase-cli = craneLib.buildPackage ( individualCrateArgs // { + # inherit (craneLib.crateNameFromCargoToml { inherit src; }) version; pname = "wordbase-cli"; cargoExtraArgs = "-p wordbase-cli"; src = fileSetForCrate ./crates/wordbase-cli; @@ -248,27 +257,12 @@ # taploExtraArgs = "--config ./taplo.toml"; }; - # Ensure that cargo-hakari is up to date - # my-workspace-hakari = craneLib.mkCargoDerivation { - # inherit src; - # pname = "my-workspace-hakari"; - # cargoArtifacts = null; - # doInstallCargoArtifacts = false; - # - # buildPhaseCargoCommand = '' - # cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date - # cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack - # cargo hakari verify - # ''; - # - # nativeBuildInputs = [ - # pkgs.cargo-hakari - # ]; - # }; }; packages = { inherit wordbase-cli; + inherit cargoArtifacts; + inherit cargoVendorDir; }; apps = {